diff --git a/claude/.claude-plugin/plugin.json b/claude/.claude-plugin/plugin.json index 395d78e..33c11ef 100644 --- a/claude/.claude-plugin/plugin.json +++ b/claude/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "chatcut", "displayName": "ChatCut", - "version": "1.10.23", + "version": "1.10.24", "description": "ChatCut plugin for editable video editing and creation workflows from Claude Code.", "author": { "name": "ChatCut", @@ -37,7 +37,7 @@ "headers": { "x-chatcut-mcp-client": "claude_code", "x-chatcut-mcp-surface": "embedded-preview", - "x-chatcut-plugin-version": "1.10.23" + "x-chatcut-plugin-version": "1.10.24" } } }, diff --git a/claude/skills/visual-analysis/SKILL.md b/claude/skills/visual-analysis/SKILL.md index d27b61c..b1d7bf3 100644 --- a/claude/skills/visual-analysis/SKILL.md +++ b/claude/skills/visual-analysis/SKILL.md @@ -1,6 +1,6 @@ --- name: visual-analysis -description: Verify composed ChatCut edits, compare source frames, locate visual changes, and build labeled contact sheets using the hosted plugin's actual frame delivery. +description: Analyze videos and references scene by scene, verify composed ChatCut edits, compare source frames, locate visual changes, and build labeled comparisons using the hosted plugin's actual frame delivery. --- # Visual Analysis @@ -9,6 +9,16 @@ Use fresh pixels to decide visual questions: whether an edit appears, when an animation changes, how a crop or overlay composes, or which source shot fits. The Codex/Claude host filesystem and ChatCut's backend workspace are separate. +## Whole-video or granular scene analysis + +For a video/reference breakdown, read +[Scene-by-scene analysis](references/scene-breakdown.md). It adds chronological +visual observations, each scene's words from the existing ChatCut transcript, +shot/screen-state detail and optional scene JSON. +Choose this path from the user's question; a focused edit check keeps the +inspection workflow below. No new provider, queue or user mode parameter is +required. Use the current host's source/timeline delivery and clock contracts. + ## Keep coordinates and subjects explicit | Subject | Tool and coordinates | What the image proves | diff --git a/claude/skills/visual-analysis/references/scene-breakdown.md b/claude/skills/visual-analysis/references/scene-breakdown.md new file mode 100644 index 0000000..13c653a --- /dev/null +++ b/claude/skills/visual-analysis/references/scene-breakdown.md @@ -0,0 +1,127 @@ +# Scene-by-scene analysis + +Use this for understanding a video, studying a reference, or producing a granular +scene/shot breakdown. Keep a focused visual check focused; a subtitle or transition +question need not generate a report about the whole video. + +## Establish the subject and evidence + +Resolve the actual source or composed timeline, requested range and useful outcome. +Reuse readable media, the existing ChatCut transcript and frames. Analyzing a local +file does not require placing it on a timeline or uploading it. For a URL, use an +available acquisition route; a thumbnail, page description or successful import +alone does not establish its content. Report inaccessible media as a source gap. + +Read duration and the relevant clock before assigning times. Original footage +uses source time; a composed edit uses its timeline clock. Follow the entrypoint's +host-specific field names and units. Keep placement, trim and rate mappings when +relating the two. The same numeric value can mean different moments on those +clocks. + +## Inspect at the level the question needs + +Begin with a bounded overview spanning the selected range, including its opening +and closing content. Use the current host's source-frame or timeline-viewer tools; +batch useful samples and divide long videos into meaningful ranges. Chapters, +transcripts and detected changes can locate candidates, but they do not prove +that the relevant visuals were inspected. + +Expand around consequential changes: a new shot, a changed software screen, +prompt or setting, an inserted example, an action's start/result, or a transition. +Inspect ordered before/during/after moments when motion matters. Read small text +from a useful full-size frame or crop. A few overview images cannot establish +every cut or a frame-exact boundary; use the available cut/adjacent-frame route +when that precision changes the answer. Do not inspect every frame by ritual. + +## Read speech from the ChatCut transcript + +This analysis assumes the media already has a ChatCut transcript. Read each +scene's timestamped words from it through the host's transcript field or view, +and put the words actually spoken in that interval into the scene's `audio`. Use +the transcript of the media's own speech; a translated or reworded caption track +does not record what was said. Do not transcribe again or reconstruct speech by +listening. If the transcript is missing, still processing or unreadable for the +range, report that gap and leave the speech unresolved; creating a transcript is +the transcription workflow's job. + +Keep the transcript's words, including uncertain or misrecognised terms. When +on-screen text shows the intended spelling, note the correction instead of +silently rewriting the quote. A sentence can cross a visual cut; assign its words +to the scene where they are spoken without inventing a word boundary. Summarise +only when the user asks for less detail, and label the summary as one. + +Listen to the audio only for tone, music, sound effects or synchronization, when +the question depends on them. Without that listening, say so and do not invent +acoustic descriptions. A transcript's music tag is evidence of the tag, not a +listening result. + +## Describe meaningful scenes + +Let visible content and the user's purpose determine segmentation. Separate +material shot/state changes; retain a coherent continuing action across transcript +sentences. A tutorial chapter can contain a presenter, a request, a software +operation and its output—those are different observations when they matter. +Group an unresolved rapid montage explicitly rather than guessing its cuts. +There is no target scene count or fixed sampling density. + +For each useful interval capture: + +- Start/end on the declared clock, at the supported precision. +- Visible subjects, setting, layout, text, action and camera/graphic movement. +- The transcript words spoken in the interval, plus any tone, music or sound + effects actually listened for. +- Framing or shot type, and the scene's role in the video when useful. + +Prefer specific observations to a repeated generic presenter description. For a +software demonstration, inspect the important screen, selected inputs, readable +settings and resulting state. Keep a creator's request separate from its result. +An output does not reveal its hidden prompt or internal production algorithm. + +Keep illegible text, inferred settings and unverified claims uncertain. Reconcile +material disagreements between narration, visible controls and results. Current +documentation can explain a tool but cannot prove the creator used it for this +clip. Recovering the full production method or creating a new video continues in +the corresponding workflow; a scene list alone does not complete either task. + +## Return useful prose or structured scenes + +Use a concise narrative/table for an ordinary question. When the user or a +downstream workflow needs JSON, the following scene fields retain the useful +granularity of Higgsfield's exposed Video Analysis results without requiring that +provider, its queue or its private prompt: + +```json +{ + "time_domain": "source", + "analyzed_range": { "start_seconds": 0, "end_seconds": 4 }, + "scenes": [ + { + "scene_number": 1, + "timestamp_start": "00:00:00.000", + "timestamp_end": "00:00:04.000", + "visual": "A close view shows a cup on a table; a title enters above it.", + "audio": "\"Here is the new cup.\" Music and effects not listened to.", + "shot_type": "Close-Up", + "label": "Product introduction" + } + ], + "evidence": { + "timing_precision": "estimated from sampled frames", + "audio_basis": "chatcut_transcript", + "coverage": "Sampled frames and transcript words across this selected range." + } +} +``` + +This is an illustrative shape, not an observation of a supplied video or a claim +about a supplied video. The seven scene keys are `scene_number`, +`timestamp_start`, `timestamp_end`, `visual`, `audio`, `shot_type` and `label`. +Keep source/asset identity, evidence locators, precision and gaps alongside the +scenes when needed. Choose useful labels for the actual content; advertising +categories should not force a tutorial or film into a marketing interpretation. + +Check chronological intervals against the actual duration and clock, including +the tail. Internally continuous times can still drift from the source. Cite the +actual frames/ranges supporting important claims. State partial coverage rather +than stretching an uninspected final scene. A completed provider job, more scenes, +valid JSON or distinct wording does not establish correct analysis. diff --git a/codex/.codex-plugin/plugin.json b/codex/.codex-plugin/plugin.json index 5d170b4..8ca44e5 100644 --- a/codex/.codex-plugin/plugin.json +++ b/codex/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "chatcut", - "version": "1.10.21", + "version": "1.10.22", "description": "ChatCut Codex plugin for editable video editing and creation workflows from Codex.", "author": { "name": "ChatCut", diff --git a/codex/.mcp.json b/codex/.mcp.json index c02fca1..fc685d3 100644 --- a/codex/.mcp.json +++ b/codex/.mcp.json @@ -3,7 +3,7 @@ "chatcut": { "http_headers": { "x-chatcut-mcp-surface": "codex", - "x-chatcut-plugin-version": "1.10.21" + "x-chatcut-plugin-version": "1.10.22" }, "oauth_resource": "https://api.chatcut.io/api/external-mcp/mcp", "url": "https://api.chatcut.io/api/external-mcp/mcp" diff --git a/codex/skills/visual-analysis/SKILL.md b/codex/skills/visual-analysis/SKILL.md index d27b61c..b1d7bf3 100644 --- a/codex/skills/visual-analysis/SKILL.md +++ b/codex/skills/visual-analysis/SKILL.md @@ -1,6 +1,6 @@ --- name: visual-analysis -description: Verify composed ChatCut edits, compare source frames, locate visual changes, and build labeled contact sheets using the hosted plugin's actual frame delivery. +description: Analyze videos and references scene by scene, verify composed ChatCut edits, compare source frames, locate visual changes, and build labeled comparisons using the hosted plugin's actual frame delivery. --- # Visual Analysis @@ -9,6 +9,16 @@ Use fresh pixels to decide visual questions: whether an edit appears, when an animation changes, how a crop or overlay composes, or which source shot fits. The Codex/Claude host filesystem and ChatCut's backend workspace are separate. +## Whole-video or granular scene analysis + +For a video/reference breakdown, read +[Scene-by-scene analysis](references/scene-breakdown.md). It adds chronological +visual observations, each scene's words from the existing ChatCut transcript, +shot/screen-state detail and optional scene JSON. +Choose this path from the user's question; a focused edit check keeps the +inspection workflow below. No new provider, queue or user mode parameter is +required. Use the current host's source/timeline delivery and clock contracts. + ## Keep coordinates and subjects explicit | Subject | Tool and coordinates | What the image proves | diff --git a/codex/skills/visual-analysis/references/scene-breakdown.md b/codex/skills/visual-analysis/references/scene-breakdown.md new file mode 100644 index 0000000..13c653a --- /dev/null +++ b/codex/skills/visual-analysis/references/scene-breakdown.md @@ -0,0 +1,127 @@ +# Scene-by-scene analysis + +Use this for understanding a video, studying a reference, or producing a granular +scene/shot breakdown. Keep a focused visual check focused; a subtitle or transition +question need not generate a report about the whole video. + +## Establish the subject and evidence + +Resolve the actual source or composed timeline, requested range and useful outcome. +Reuse readable media, the existing ChatCut transcript and frames. Analyzing a local +file does not require placing it on a timeline or uploading it. For a URL, use an +available acquisition route; a thumbnail, page description or successful import +alone does not establish its content. Report inaccessible media as a source gap. + +Read duration and the relevant clock before assigning times. Original footage +uses source time; a composed edit uses its timeline clock. Follow the entrypoint's +host-specific field names and units. Keep placement, trim and rate mappings when +relating the two. The same numeric value can mean different moments on those +clocks. + +## Inspect at the level the question needs + +Begin with a bounded overview spanning the selected range, including its opening +and closing content. Use the current host's source-frame or timeline-viewer tools; +batch useful samples and divide long videos into meaningful ranges. Chapters, +transcripts and detected changes can locate candidates, but they do not prove +that the relevant visuals were inspected. + +Expand around consequential changes: a new shot, a changed software screen, +prompt or setting, an inserted example, an action's start/result, or a transition. +Inspect ordered before/during/after moments when motion matters. Read small text +from a useful full-size frame or crop. A few overview images cannot establish +every cut or a frame-exact boundary; use the available cut/adjacent-frame route +when that precision changes the answer. Do not inspect every frame by ritual. + +## Read speech from the ChatCut transcript + +This analysis assumes the media already has a ChatCut transcript. Read each +scene's timestamped words from it through the host's transcript field or view, +and put the words actually spoken in that interval into the scene's `audio`. Use +the transcript of the media's own speech; a translated or reworded caption track +does not record what was said. Do not transcribe again or reconstruct speech by +listening. If the transcript is missing, still processing or unreadable for the +range, report that gap and leave the speech unresolved; creating a transcript is +the transcription workflow's job. + +Keep the transcript's words, including uncertain or misrecognised terms. When +on-screen text shows the intended spelling, note the correction instead of +silently rewriting the quote. A sentence can cross a visual cut; assign its words +to the scene where they are spoken without inventing a word boundary. Summarise +only when the user asks for less detail, and label the summary as one. + +Listen to the audio only for tone, music, sound effects or synchronization, when +the question depends on them. Without that listening, say so and do not invent +acoustic descriptions. A transcript's music tag is evidence of the tag, not a +listening result. + +## Describe meaningful scenes + +Let visible content and the user's purpose determine segmentation. Separate +material shot/state changes; retain a coherent continuing action across transcript +sentences. A tutorial chapter can contain a presenter, a request, a software +operation and its output—those are different observations when they matter. +Group an unresolved rapid montage explicitly rather than guessing its cuts. +There is no target scene count or fixed sampling density. + +For each useful interval capture: + +- Start/end on the declared clock, at the supported precision. +- Visible subjects, setting, layout, text, action and camera/graphic movement. +- The transcript words spoken in the interval, plus any tone, music or sound + effects actually listened for. +- Framing or shot type, and the scene's role in the video when useful. + +Prefer specific observations to a repeated generic presenter description. For a +software demonstration, inspect the important screen, selected inputs, readable +settings and resulting state. Keep a creator's request separate from its result. +An output does not reveal its hidden prompt or internal production algorithm. + +Keep illegible text, inferred settings and unverified claims uncertain. Reconcile +material disagreements between narration, visible controls and results. Current +documentation can explain a tool but cannot prove the creator used it for this +clip. Recovering the full production method or creating a new video continues in +the corresponding workflow; a scene list alone does not complete either task. + +## Return useful prose or structured scenes + +Use a concise narrative/table for an ordinary question. When the user or a +downstream workflow needs JSON, the following scene fields retain the useful +granularity of Higgsfield's exposed Video Analysis results without requiring that +provider, its queue or its private prompt: + +```json +{ + "time_domain": "source", + "analyzed_range": { "start_seconds": 0, "end_seconds": 4 }, + "scenes": [ + { + "scene_number": 1, + "timestamp_start": "00:00:00.000", + "timestamp_end": "00:00:04.000", + "visual": "A close view shows a cup on a table; a title enters above it.", + "audio": "\"Here is the new cup.\" Music and effects not listened to.", + "shot_type": "Close-Up", + "label": "Product introduction" + } + ], + "evidence": { + "timing_precision": "estimated from sampled frames", + "audio_basis": "chatcut_transcript", + "coverage": "Sampled frames and transcript words across this selected range." + } +} +``` + +This is an illustrative shape, not an observation of a supplied video or a claim +about a supplied video. The seven scene keys are `scene_number`, +`timestamp_start`, `timestamp_end`, `visual`, `audio`, `shot_type` and `label`. +Keep source/asset identity, evidence locators, precision and gaps alongside the +scenes when needed. Choose useful labels for the actual content; advertising +categories should not force a tutorial or film into a marketing interpretation. + +Check chronological intervals against the actual duration and clock, including +the tail. Internally continuous times can still drift from the source. Cite the +actual frames/ranges supporting important claims. State partial coverage rather +than stretching an uninspected final scene. A completed provider job, more scenes, +valid JSON or distinct wording does not establish correct analysis. diff --git a/grok/.cursor-plugin/plugin.json b/grok/.cursor-plugin/plugin.json index 418952e..2a48198 100644 --- a/grok/.cursor-plugin/plugin.json +++ b/grok/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "chatcut", - "version": "1.10.17", + "version": "1.10.18", "description": "Let Grok Bot agents create and edit videos in ChatCut through natural-language requests, with editable timelines, captions, motion graphics, media generation, and export.", "author": { "name": "ChatCut", diff --git a/grok/mcp.json b/grok/mcp.json index 9f00808..f01d470 100644 --- a/grok/mcp.json +++ b/grok/mcp.json @@ -4,7 +4,7 @@ "url": "https://api.chatcut.io/api/external-mcp/mcp", "headers": { "x-chatcut-mcp-surface": "embedded-preview", - "x-chatcut-plugin-version": "1.10.17" + "x-chatcut-plugin-version": "1.10.18" } } } diff --git a/grok/skills/verification/SKILL.md b/grok/skills/verification/SKILL.md index 18b7273..0e67361 100644 --- a/grok/skills/verification/SKILL.md +++ b/grok/skills/verification/SKILL.md @@ -5,6 +5,10 @@ description: Hosted ChatCut plugin sessions only (the `chatcut` MCP server). If # Verification +For a video/reference breakdown or detailed frame analysis, use the bundled +[visual-analysis skill](../visual-analysis/SKILL.md), then continue the checks +below when verifying an edit. + Use both structural and visual evidence when the requested result is visible: 1. Re-read the affected scope with `read_project`, `preview_timeline`, `inspect_item`, `browse_assets`, or `inspect_asset` as appropriate. Use the latest response for ids, tracks, timing, and readiness. diff --git a/grok/skills/visual-analysis/SKILL.md b/grok/skills/visual-analysis/SKILL.md new file mode 100644 index 0000000..8c4084a --- /dev/null +++ b/grok/skills/visual-analysis/SKILL.md @@ -0,0 +1,106 @@ +--- +name: visual-analysis +description: Analyze videos and references scene by scene, verify composed ChatCut edits, compare source frames, locate visual changes, and build labeled comparisons using the hosted plugin's actual frame delivery. +--- + +# Visual Analysis + +Use fresh pixels to decide visual questions: whether an edit appears, when an +animation changes, how a crop or overlay composes, or which source shot fits. +The agent host filesystem and ChatCut's backend workspace are separate. + +## Whole-video or granular scene analysis + +For a video/reference breakdown, read +[Scene-by-scene analysis](references/scene-breakdown.md). It adds chronological +visual observations, each scene's words from the existing ChatCut transcript, +shot/screen-state detail and optional scene JSON. +Choose this path from the user's question; a focused edit check keeps the +inspection workflow below. No new provider, queue or user mode parameter is +required. Use the current host's source/timeline delivery and clock contracts. + +## Keep coordinates and subjects explicit + +| Subject | Tool and coordinates | What the image proves | +| ----------------------------------- | --------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| Placed timeline composition | `preview_timeline`, `views:["viewer"]`, `viewerFrames` in timeline frames | Clips with their placement, crop, captions, overlays, effects, and framing | +| Original video / Motion Graphic | `inspect_asset`, `sourceFrameCount` for an overview or `sourceTimesMs` in source milliseconds | The source media at those positions | +| Original image / SVG | `inspect_asset` with its default image preview | The source image, without timeline placement | +| Readable original file in this host | The host's local inspection tools | Source bytes; no temporary timeline or cloud upload is needed | + +Use `timelineId` when scope could be ambiguous. Viewer metadata reports the +actual `frame`, `timelineTimeMs`, and FPS. Source metadata reports `sourceTimeMs`. +Do not label a source timestamp as timeline time. Read a placed item's start, +source offset, playback rate and timeline FPS before mapping between them. + +Timeline ranges are half-open `[fromFrame,toFrame)`. Frames at `toFrame` or the +duration are outside the renderable range. The hosted tool may skip invalid +requested moments and report adjustments; it fails when none are renderable. +Check the returned frames rather than assuming every request was rendered. +`sourceTimesMs` and `sourceFrameCount` are alternatives; bounded transcript +context uses the separate `transcriptRangesMs` field. + +## Read the actual delivery + +Preview tools can return an inline image, temporary per-frame `resource_link` +URIs, or written workspace paths on a workspace-backed runtime. Hosted plugins +have no backend workspace. Inspect inline images directly when the host shows +them. For image links, use the full returned URI with a host-supported image +reader, or an allowed download into the host's own analysis directory. Never +infer local files from names such as `timeline-frames/...`, and never treat a +backend `workspacePath` as a path on the user's computer. + +If rendering is unavailable, read its diagnostic. Source-frame requests on +this connector need the target project open in a capable editor; there is no +automatic cloud fallback. Inspect a readable original locally when that is the +subject, or surface the returned editor URL for composed proof. A metadata read +or successful tool call without visible pixels does not establish appearance. + +## Narrow a change + +Start with a frame before and after the suspected change. Sample a few moments +between them, then refine the adjacent pair that brackets the difference. +Stop at adjacent timeline frames when exact onset matters. For source footage, +refine source milliseconds instead. An overview proves only sampled moments. + +Read full frames before cropping. Use the same crop rectangle and scale for +comparisons, and label the returned frame/time, timeline or source identity. +Keep chronological order left-to-right, then top-to-bottom. Render fresh +evidence after editing; an old contact sheet describes the old state. + +## Straightening across shots + +A clip can contain different camera angles. Choose a reliable level reference +for each affected shot; a doorway viewed in perspective is not enough to infer +camera roll. Apply one rotation to the whole clip only when it suits all shots; +otherwise limit the correction to the intended shot or range and preserve the +others. If the intended reference is unclear, clarify that choice before editing. + +Compare matching timeline frames before and after the correction, including +shots that should stay unchanged. Check level, framing and cropped edges together. +Await the edit before requesting its preview, and inspect fresh pixels after an +undo too. Report straightening or restoration only to the extent those frames +verify it. + +## Contact sheets from host-local images + +Use the tool's returned labeled sheet when available. For custom comparisons, +the shared [concat-images helper](scripts/concat-images.mjs) accepts images that +already exist on the host. It needs Node and an available ImageMagick command; +if those are absent, use the host's existing image tools or inspect frames +individually. This skill does not require a new package installation. + +```bash +node /scripts/concat-images.mjs \ + --output /title-change.jpg --columns 3 \ + --caption "timeline frame 120" --caption "timeline frame 123" \ + --caption "timeline frame 126" \ + /frame-120.jpg /frame-123.jpg \ + /frame-126.jpg +``` + +`--crop WIDTHxHEIGHT+X+Y` applies a common pixel crop; `--tile-width` limits tile +size. Read the helper's JSON and inspect the resulting `outputPath` using the +host's image viewer. Label strips are analysis annotations, not video content. +Analysis images stay separate from project media; import one through +[asset-import](../asset-import/SKILL.md) only if the user wants to keep or use it. diff --git a/grok/skills/visual-analysis/references/scene-breakdown.md b/grok/skills/visual-analysis/references/scene-breakdown.md new file mode 100644 index 0000000..13c653a --- /dev/null +++ b/grok/skills/visual-analysis/references/scene-breakdown.md @@ -0,0 +1,127 @@ +# Scene-by-scene analysis + +Use this for understanding a video, studying a reference, or producing a granular +scene/shot breakdown. Keep a focused visual check focused; a subtitle or transition +question need not generate a report about the whole video. + +## Establish the subject and evidence + +Resolve the actual source or composed timeline, requested range and useful outcome. +Reuse readable media, the existing ChatCut transcript and frames. Analyzing a local +file does not require placing it on a timeline or uploading it. For a URL, use an +available acquisition route; a thumbnail, page description or successful import +alone does not establish its content. Report inaccessible media as a source gap. + +Read duration and the relevant clock before assigning times. Original footage +uses source time; a composed edit uses its timeline clock. Follow the entrypoint's +host-specific field names and units. Keep placement, trim and rate mappings when +relating the two. The same numeric value can mean different moments on those +clocks. + +## Inspect at the level the question needs + +Begin with a bounded overview spanning the selected range, including its opening +and closing content. Use the current host's source-frame or timeline-viewer tools; +batch useful samples and divide long videos into meaningful ranges. Chapters, +transcripts and detected changes can locate candidates, but they do not prove +that the relevant visuals were inspected. + +Expand around consequential changes: a new shot, a changed software screen, +prompt or setting, an inserted example, an action's start/result, or a transition. +Inspect ordered before/during/after moments when motion matters. Read small text +from a useful full-size frame or crop. A few overview images cannot establish +every cut or a frame-exact boundary; use the available cut/adjacent-frame route +when that precision changes the answer. Do not inspect every frame by ritual. + +## Read speech from the ChatCut transcript + +This analysis assumes the media already has a ChatCut transcript. Read each +scene's timestamped words from it through the host's transcript field or view, +and put the words actually spoken in that interval into the scene's `audio`. Use +the transcript of the media's own speech; a translated or reworded caption track +does not record what was said. Do not transcribe again or reconstruct speech by +listening. If the transcript is missing, still processing or unreadable for the +range, report that gap and leave the speech unresolved; creating a transcript is +the transcription workflow's job. + +Keep the transcript's words, including uncertain or misrecognised terms. When +on-screen text shows the intended spelling, note the correction instead of +silently rewriting the quote. A sentence can cross a visual cut; assign its words +to the scene where they are spoken without inventing a word boundary. Summarise +only when the user asks for less detail, and label the summary as one. + +Listen to the audio only for tone, music, sound effects or synchronization, when +the question depends on them. Without that listening, say so and do not invent +acoustic descriptions. A transcript's music tag is evidence of the tag, not a +listening result. + +## Describe meaningful scenes + +Let visible content and the user's purpose determine segmentation. Separate +material shot/state changes; retain a coherent continuing action across transcript +sentences. A tutorial chapter can contain a presenter, a request, a software +operation and its output—those are different observations when they matter. +Group an unresolved rapid montage explicitly rather than guessing its cuts. +There is no target scene count or fixed sampling density. + +For each useful interval capture: + +- Start/end on the declared clock, at the supported precision. +- Visible subjects, setting, layout, text, action and camera/graphic movement. +- The transcript words spoken in the interval, plus any tone, music or sound + effects actually listened for. +- Framing or shot type, and the scene's role in the video when useful. + +Prefer specific observations to a repeated generic presenter description. For a +software demonstration, inspect the important screen, selected inputs, readable +settings and resulting state. Keep a creator's request separate from its result. +An output does not reveal its hidden prompt or internal production algorithm. + +Keep illegible text, inferred settings and unverified claims uncertain. Reconcile +material disagreements between narration, visible controls and results. Current +documentation can explain a tool but cannot prove the creator used it for this +clip. Recovering the full production method or creating a new video continues in +the corresponding workflow; a scene list alone does not complete either task. + +## Return useful prose or structured scenes + +Use a concise narrative/table for an ordinary question. When the user or a +downstream workflow needs JSON, the following scene fields retain the useful +granularity of Higgsfield's exposed Video Analysis results without requiring that +provider, its queue or its private prompt: + +```json +{ + "time_domain": "source", + "analyzed_range": { "start_seconds": 0, "end_seconds": 4 }, + "scenes": [ + { + "scene_number": 1, + "timestamp_start": "00:00:00.000", + "timestamp_end": "00:00:04.000", + "visual": "A close view shows a cup on a table; a title enters above it.", + "audio": "\"Here is the new cup.\" Music and effects not listened to.", + "shot_type": "Close-Up", + "label": "Product introduction" + } + ], + "evidence": { + "timing_precision": "estimated from sampled frames", + "audio_basis": "chatcut_transcript", + "coverage": "Sampled frames and transcript words across this selected range." + } +} +``` + +This is an illustrative shape, not an observation of a supplied video or a claim +about a supplied video. The seven scene keys are `scene_number`, +`timestamp_start`, `timestamp_end`, `visual`, `audio`, `shot_type` and `label`. +Keep source/asset identity, evidence locators, precision and gaps alongside the +scenes when needed. Choose useful labels for the actual content; advertising +categories should not force a tutorial or film into a marketing interpretation. + +Check chronological intervals against the actual duration and clock, including +the tail. Internally continuous times can still drift from the source. Cite the +actual frames/ranges supporting important claims. State partial coverage rather +than stretching an uninspected final scene. A completed provider job, more scenes, +valid JSON or distinct wording does not establish correct analysis. diff --git a/grok/skills/visual-analysis/scripts/concat-images.mjs b/grok/skills/visual-analysis/scripts/concat-images.mjs new file mode 100644 index 0000000..56be49b --- /dev/null +++ b/grok/skills/visual-analysis/scripts/concat-images.mjs @@ -0,0 +1,305 @@ +#!/usr/bin/env node + +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { basename, dirname, extname, join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; + +const parsed = parseArgs(process.argv.slice(2)); +const outputPath = resolve(parsed.output); +const outputExtension = extname(outputPath).toLowerCase(); + +if (![".jpg", ".jpeg", ".png", ".webp"].includes(outputExtension)) { + fail("--output must end in .jpg, .jpeg, .png, or .webp"); +} +if (parsed.inputs.length === 0 || parsed.inputs.length > 25) { + fail("Pass between 1 and 25 input images"); +} +if (parsed.captions.length && parsed.captions.length !== parsed.inputs.length) { + fail("Pass exactly one --caption per input image, or omit captions entirely"); +} + +const inputs = parsed.inputs.map((input) => resolve(input)); +for (const input of inputs) { + if (!existsSync(input)) fail(`Input image does not exist: ${input}`); +} + +mkdirSync(dirname(outputPath), { recursive: true }); +const temporaryDirectory = mkdtempSync( + join(dirname(outputPath), ".concat-images-"), +); +const imageMagick = resolveImageMagick(); +// Some ImageMagick builds cannot discover installed fonts without fontconfig. +const captions = parsed.captions.length + ? parsed.captions + : inputs.map(defaultCaption); +const captionFont = [ + ...(captions.every((caption) => /^[\x00-\x7F]*$/.test(caption)) + ? [ + "/System/Library/Fonts/Menlo.ttc", + "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", + ] + : []), + "/System/Library/Fonts/Supplemental/Arial Unicode.ttf", + "/System/Library/Fonts/Supplemental/Arial.ttf", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + join(process.env.WINDIR ?? "C:\\Windows", "Fonts", "arial.ttf"), +].find((font) => existsSync(font)); + +try { + const prepared = inputs.map((input, index) => { + const preparedPath = join(temporaryDirectory, `${index}.png`); + const args = imageMagick.prefix.concat([ + input, + ...(parsed.crop ? ["-crop", parsed.crop, "+repage"] : []), + "-auto-orient", + "+profile", + "icc", + "-resize", + `${parsed.tileWidth}x${parsed.tileWidth}>`, + preparedPath, + ]); + run(imageMagick.command, args, `prepare ${input}`); + return preparedPath; + }); + + const dimensions = run( + imageMagick.command, + imageMagick.prefix.concat(prepared, ["-format", "%w %h\n", "info:"]), + "read image dimensions", + ) + .trim() + .split("\n") + .map((line) => line.split(" ").map(Number)); + const aspect = + Math.max(...dimensions.map(([width]) => width)) / + Math.max(...dimensions.map(([, height]) => height)); + const captionHeight = parsed.noCaptions ? 0 : 28; + let columns = parsed.columns ?? 1; + let tileWidth = 0; + for ( + let candidate = 1; + candidate <= Math.max(inputs.length, parsed.columns ?? 1); + candidate++ + ) { + if (parsed.columns && candidate !== parsed.columns) continue; + const rows = Math.ceil(inputs.length / candidate); + const width = Math.floor( + Math.min( + parsed.tileWidth, + 1568 / candidate - 2 * parsed.gap, + (Math.floor(1568 / rows) - 2 * parsed.gap - captionHeight) * aspect, + ), + ); + if (width > tileWidth) { + tileWidth = width; + columns = candidate; + } + } + if (tileWidth < 1) fail("The requested columns/gap leave no room for images"); + const tileHeight = Math.max(1, Math.floor(tileWidth / aspect)); + const captioned = prepared.map((input, index) => { + const output = join(temporaryDirectory, `captioned-${index}.png`); + run( + imageMagick.command, + imageMagick.prefix.concat([ + input, + "-resize", + `${tileWidth}x${tileHeight}`, + "-background", + "#000000", + "-gravity", + "center", + "-extent", + `${tileWidth}x${tileHeight}`, + ...(parsed.noCaptions + ? [] + : [ + "(", + "-size", + `${tileWidth}x${captionHeight}`, + "xc:#eeeeee", + ...(captionFont ? ["-font", captionFont] : []), + "-fill", + "#111111", + "-stroke", + "none", + "-pointsize", + "18", + "-gravity", + "west", + "-annotate", + "+5+0", + captions[index], + ")", + "-append", + ]), + output, + ]), + "add caption below image", + ); + return output; + }); + const montageArgs = imageMagick.montagePrefix.concat([ + ...(captionFont ? ["-font", captionFont] : []), + ...captioned, + "-tile", + `${columns}x`, + "-geometry", + `+${parsed.gap}+${parsed.gap}`, + "-background", + "#eeeeee", + outputPath, + ]); + run(imageMagick.command, montageArgs, "compose contact sheet"); + + process.stdout.write( + `${JSON.stringify( + { + columns, + crop: parsed.crop ?? null, + inputs: inputs.map((input, index) => ({ + caption: captions[index], + path: input, + })), + ...(parsed.noCaptions + ? {} + : { + note: "The light-gray strips below the images contain tool-added position labels, not video content.", + }), + width: columns * (tileWidth + 2 * parsed.gap), + height: + Math.ceil(inputs.length / columns) * + (tileHeight + captionHeight + 2 * parsed.gap), + outputPath, + rows: Math.ceil(inputs.length / columns), + }, + null, + 2, + )}\n`, + ); +} finally { + rmSync(temporaryDirectory, { force: true, recursive: true }); +} + +// Frame filenames carry the exact coordinate returned by the preview tools. +function defaultCaption(input) { + const name = basename(input, extname(input)); + const timeline = + name.match(/^timeline-(\d+)f-\d+ms-\d+$/) ?? + name.match(/^timeline-frame-(\d+)$/); + if (timeline) return `f${timeline[1]}`; + const source = + name.match(/^source-(\d+)ms-\d+$/) ?? name.match(/^asset-frame-(\d+)ms$/); + if (source) return `${(Number(source[1]) / 1000).toFixed(3)}s`; + return basename(input); +} + +function parseArgs(args) { + const result = { + noCaptions: false, + captions: [], + columns: undefined, + crop: undefined, + gap: 0, + inputs: [], + output: undefined, + tileWidth: 720, + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--no-captions") result.noCaptions = true; + else if (arg === "--caption") + result.captions.push(requiredValue(args, ++index, arg)); + else if (arg === "--columns") + result.columns = boundedInteger( + requiredValue(args, ++index, arg), + arg, + 1, + 9, + ); + else if (arg === "--crop") result.crop = requiredValue(args, ++index, arg); + else if (arg === "--gap") + result.gap = boundedInteger( + requiredValue(args, ++index, arg), + arg, + 0, + 100, + ); + else if (arg === "--output") + result.output = requiredValue(args, ++index, arg); + else if (arg === "--tile-width") + result.tileWidth = boundedInteger( + requiredValue(args, ++index, arg), + arg, + 100, + 2000, + ); + else if (arg === "--help" || arg === "-h") usage(0); + else if (arg.startsWith("-")) fail(`Unknown option: ${arg}`); + else result.inputs.push(arg); + } + if (!result.output) fail("--output is required"); + if (result.crop && !/^\d+x\d+\+\d+\+\d+$/.test(result.crop)) { + fail("--crop must use WIDTHxHEIGHT+X+Y pixel geometry"); + } + return result; +} + +function resolveImageMagick() { + if (commandExists("magick")) { + return { command: "magick", montagePrefix: ["montage"], prefix: [] }; + } + if (commandExists("convert") && commandExists("montage")) { + return { command: "convert", montagePrefix: [], prefix: [] }; + } + fail("ImageMagick is unavailable; expected magick or convert + montage"); +} + +function commandExists(command) { + return ( + spawnSync("sh", ["-c", `command -v ${command}`], { + stdio: "ignore", + }).status === 0 + ); +} + +function run(command, args, step) { + const executable = + step === "compose contact sheet" && command === "convert" + ? "montage" + : command; + const result = spawnSync(executable, args, { encoding: "utf8" }); + if (result.status !== 0) { + fail( + `${step} failed: ${(result.stderr || result.stdout || "unknown error").trim()}`, + ); + } + return result.stdout; +} + +function requiredValue(args, index, option) { + const value = args[index]; + if (!value) fail(`${option} requires a value`); + return value; +} + +function boundedInteger(value, option, minimum, maximum) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) { + fail(`${option} must be an integer from ${minimum} to ${maximum}`); + } + return parsed; +} + +function fail(message) { + process.stderr.write(`concat-images: ${message}\n`); + process.exit(1); +} + +function usage(exitCode) { + process.stdout.write( + "Usage: concat-images.mjs --output [--columns N] [--crop WxH+X+Y] [--caption text ...] \n", + ); + process.exit(exitCode); +} diff --git a/kimi/.mcp.json b/kimi/.mcp.json index c0c2ee2..c7e48c8 100644 --- a/kimi/.mcp.json +++ b/kimi/.mcp.json @@ -8,7 +8,7 @@ "http_headers": { "x-chatcut-mcp-client": "kimi_work", "x-chatcut-mcp-surface": "embedded-preview", - "x-chatcut-plugin-version": "1.10.17" + "x-chatcut-plugin-version": "1.10.18" }, "note": "Official ChatCut remote MCP endpoint. OAuth 2.0 with dynamic client registration (DCR) + PKCE; the server derives userId from the token.", "authorization": "OAuth 2.0 with dynamic client registration", diff --git a/kimi/kimi.plugin.json b/kimi/kimi.plugin.json index a8d16b2..52a9c46 100644 --- a/kimi/kimi.plugin.json +++ b/kimi/kimi.plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://catalog.msh.team/misc/kimi.plugin.schema.json", "name": "chatcut-kimi-work", - "version": "1.10.17", + "version": "1.10.18", "description": "ChatCut plugin for Kimi Work: let the agent edit speech, create motion graphics and media, and deliver editable video projects in the live in-app editor.", "author": "ChatCut", "homepage": "https://chatcut.io", @@ -40,7 +40,7 @@ "headers": { "x-chatcut-mcp-client": "kimi_work", "x-chatcut-mcp-surface": "embedded-preview", - "x-chatcut-plugin-version": "1.10.17" + "x-chatcut-plugin-version": "1.10.18" } } } diff --git a/kimi/skills/chatcut-verification-guide/SKILL.md b/kimi/skills/chatcut-verification-guide/SKILL.md index efe5533..d412985 100644 --- a/kimi/skills/chatcut-verification-guide/SKILL.md +++ b/kimi/skills/chatcut-verification-guide/SKILL.md @@ -5,6 +5,10 @@ description: Use after ChatCut edits, imports, captions, generation, or exports # ChatCut Verification Guide +For a video/reference breakdown or detailed frame analysis, use the bundled +[visual-analysis skill](../visual-analysis/SKILL.md), then continue the checks +below when verifying an edit. + Verify through ChatCut project state first and the in-app browser second. ## Checks diff --git a/kimi/skills/visual-analysis/SKILL.md b/kimi/skills/visual-analysis/SKILL.md new file mode 100644 index 0000000..346c2f6 --- /dev/null +++ b/kimi/skills/visual-analysis/SKILL.md @@ -0,0 +1,106 @@ +--- +name: visual-analysis +description: Analyze videos and references scene by scene, verify composed ChatCut edits, compare source frames, locate visual changes, and build labeled comparisons using the hosted plugin's actual frame delivery. +--- + +# Visual Analysis + +Use fresh pixels to decide visual questions: whether an edit appears, when an +animation changes, how a crop or overlay composes, or which source shot fits. +The agent host filesystem and ChatCut's backend workspace are separate. + +## Whole-video or granular scene analysis + +For a video/reference breakdown, read +[Scene-by-scene analysis](references/scene-breakdown.md). It adds chronological +visual observations, each scene's words from the existing ChatCut transcript, +shot/screen-state detail and optional scene JSON. +Choose this path from the user's question; a focused edit check keeps the +inspection workflow below. No new provider, queue or user mode parameter is +required. Use the current host's source/timeline delivery and clock contracts. + +## Keep coordinates and subjects explicit + +| Subject | Tool and coordinates | What the image proves | +| ----------------------------------- | --------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| Placed timeline composition | `preview_timeline`, `views:["viewer"]`, `viewerFrames` in timeline frames | Clips with their placement, crop, captions, overlays, effects, and framing | +| Original video / Motion Graphic | `inspect_asset`, `sourceFrameCount` for an overview or `sourceTimesMs` in source milliseconds | The source media at those positions | +| Original image / SVG | `inspect_asset` with its default image preview | The source image, without timeline placement | +| Readable original file in this host | The host's local inspection tools | Source bytes; no temporary timeline or cloud upload is needed | + +Use `timelineId` when scope could be ambiguous. Viewer metadata reports the +actual `frame`, `timelineTimeMs`, and FPS. Source metadata reports `sourceTimeMs`. +Do not label a source timestamp as timeline time. Read a placed item's start, +source offset, playback rate and timeline FPS before mapping between them. + +Timeline ranges are half-open `[fromFrame,toFrame)`. Frames at `toFrame` or the +duration are outside the renderable range. The hosted tool may skip invalid +requested moments and report adjustments; it fails when none are renderable. +Check the returned frames rather than assuming every request was rendered. +`sourceTimesMs` and `sourceFrameCount` are alternatives; bounded transcript +context uses the separate `transcriptRangesMs` field. + +## Read the actual delivery + +Preview tools can return an inline image, temporary per-frame `resource_link` +URIs, or written workspace paths on a workspace-backed runtime. Hosted plugins +have no backend workspace. Inspect inline images directly when the host shows +them. For image links, use the full returned URI with a host-supported image +reader, or an allowed download into the host's own analysis directory. Never +infer local files from names such as `timeline-frames/...`, and never treat a +backend `workspacePath` as a path on the user's computer. + +If rendering is unavailable, read its diagnostic. Source-frame requests on +this connector need the target project open in a capable editor; there is no +automatic cloud fallback. Inspect a readable original locally when that is the +subject, or surface the returned editor URL for composed proof. A metadata read +or successful tool call without visible pixels does not establish appearance. + +## Narrow a change + +Start with a frame before and after the suspected change. Sample a few moments +between them, then refine the adjacent pair that brackets the difference. +Stop at adjacent timeline frames when exact onset matters. For source footage, +refine source milliseconds instead. An overview proves only sampled moments. + +Read full frames before cropping. Use the same crop rectangle and scale for +comparisons, and label the returned frame/time, timeline or source identity. +Keep chronological order left-to-right, then top-to-bottom. Render fresh +evidence after editing; an old contact sheet describes the old state. + +## Straightening across shots + +A clip can contain different camera angles. Choose a reliable level reference +for each affected shot; a doorway viewed in perspective is not enough to infer +camera roll. Apply one rotation to the whole clip only when it suits all shots; +otherwise limit the correction to the intended shot or range and preserve the +others. If the intended reference is unclear, clarify that choice before editing. + +Compare matching timeline frames before and after the correction, including +shots that should stay unchanged. Check level, framing and cropped edges together. +Await the edit before requesting its preview, and inspect fresh pixels after an +undo too. Report straightening or restoration only to the extent those frames +verify it. + +## Contact sheets from host-local images + +Use the tool's returned labeled sheet when available. For custom comparisons, +the shared [concat-images helper](scripts/concat-images.mjs) accepts images that +already exist on the host. It needs Node and an available ImageMagick command; +if those are absent, use the host's existing image tools or inspect frames +individually. This skill does not require a new package installation. + +```bash +node /scripts/concat-images.mjs \ + --output /title-change.jpg --columns 3 \ + --caption "timeline frame 120" --caption "timeline frame 123" \ + --caption "timeline frame 126" \ + /frame-120.jpg /frame-123.jpg \ + /frame-126.jpg +``` + +`--crop WIDTHxHEIGHT+X+Y` applies a common pixel crop; `--tile-width` limits tile +size. Read the helper's JSON and inspect the resulting `outputPath` using the +host's image viewer. Label strips are analysis annotations, not video content. +Analysis images stay separate from project media; import one through +[media guide](../chatcut-media-guide/SKILL.md) only if the user wants to keep or use it. diff --git a/kimi/skills/visual-analysis/references/scene-breakdown.md b/kimi/skills/visual-analysis/references/scene-breakdown.md new file mode 100644 index 0000000..13c653a --- /dev/null +++ b/kimi/skills/visual-analysis/references/scene-breakdown.md @@ -0,0 +1,127 @@ +# Scene-by-scene analysis + +Use this for understanding a video, studying a reference, or producing a granular +scene/shot breakdown. Keep a focused visual check focused; a subtitle or transition +question need not generate a report about the whole video. + +## Establish the subject and evidence + +Resolve the actual source or composed timeline, requested range and useful outcome. +Reuse readable media, the existing ChatCut transcript and frames. Analyzing a local +file does not require placing it on a timeline or uploading it. For a URL, use an +available acquisition route; a thumbnail, page description or successful import +alone does not establish its content. Report inaccessible media as a source gap. + +Read duration and the relevant clock before assigning times. Original footage +uses source time; a composed edit uses its timeline clock. Follow the entrypoint's +host-specific field names and units. Keep placement, trim and rate mappings when +relating the two. The same numeric value can mean different moments on those +clocks. + +## Inspect at the level the question needs + +Begin with a bounded overview spanning the selected range, including its opening +and closing content. Use the current host's source-frame or timeline-viewer tools; +batch useful samples and divide long videos into meaningful ranges. Chapters, +transcripts and detected changes can locate candidates, but they do not prove +that the relevant visuals were inspected. + +Expand around consequential changes: a new shot, a changed software screen, +prompt or setting, an inserted example, an action's start/result, or a transition. +Inspect ordered before/during/after moments when motion matters. Read small text +from a useful full-size frame or crop. A few overview images cannot establish +every cut or a frame-exact boundary; use the available cut/adjacent-frame route +when that precision changes the answer. Do not inspect every frame by ritual. + +## Read speech from the ChatCut transcript + +This analysis assumes the media already has a ChatCut transcript. Read each +scene's timestamped words from it through the host's transcript field or view, +and put the words actually spoken in that interval into the scene's `audio`. Use +the transcript of the media's own speech; a translated or reworded caption track +does not record what was said. Do not transcribe again or reconstruct speech by +listening. If the transcript is missing, still processing or unreadable for the +range, report that gap and leave the speech unresolved; creating a transcript is +the transcription workflow's job. + +Keep the transcript's words, including uncertain or misrecognised terms. When +on-screen text shows the intended spelling, note the correction instead of +silently rewriting the quote. A sentence can cross a visual cut; assign its words +to the scene where they are spoken without inventing a word boundary. Summarise +only when the user asks for less detail, and label the summary as one. + +Listen to the audio only for tone, music, sound effects or synchronization, when +the question depends on them. Without that listening, say so and do not invent +acoustic descriptions. A transcript's music tag is evidence of the tag, not a +listening result. + +## Describe meaningful scenes + +Let visible content and the user's purpose determine segmentation. Separate +material shot/state changes; retain a coherent continuing action across transcript +sentences. A tutorial chapter can contain a presenter, a request, a software +operation and its output—those are different observations when they matter. +Group an unresolved rapid montage explicitly rather than guessing its cuts. +There is no target scene count or fixed sampling density. + +For each useful interval capture: + +- Start/end on the declared clock, at the supported precision. +- Visible subjects, setting, layout, text, action and camera/graphic movement. +- The transcript words spoken in the interval, plus any tone, music or sound + effects actually listened for. +- Framing or shot type, and the scene's role in the video when useful. + +Prefer specific observations to a repeated generic presenter description. For a +software demonstration, inspect the important screen, selected inputs, readable +settings and resulting state. Keep a creator's request separate from its result. +An output does not reveal its hidden prompt or internal production algorithm. + +Keep illegible text, inferred settings and unverified claims uncertain. Reconcile +material disagreements between narration, visible controls and results. Current +documentation can explain a tool but cannot prove the creator used it for this +clip. Recovering the full production method or creating a new video continues in +the corresponding workflow; a scene list alone does not complete either task. + +## Return useful prose or structured scenes + +Use a concise narrative/table for an ordinary question. When the user or a +downstream workflow needs JSON, the following scene fields retain the useful +granularity of Higgsfield's exposed Video Analysis results without requiring that +provider, its queue or its private prompt: + +```json +{ + "time_domain": "source", + "analyzed_range": { "start_seconds": 0, "end_seconds": 4 }, + "scenes": [ + { + "scene_number": 1, + "timestamp_start": "00:00:00.000", + "timestamp_end": "00:00:04.000", + "visual": "A close view shows a cup on a table; a title enters above it.", + "audio": "\"Here is the new cup.\" Music and effects not listened to.", + "shot_type": "Close-Up", + "label": "Product introduction" + } + ], + "evidence": { + "timing_precision": "estimated from sampled frames", + "audio_basis": "chatcut_transcript", + "coverage": "Sampled frames and transcript words across this selected range." + } +} +``` + +This is an illustrative shape, not an observation of a supplied video or a claim +about a supplied video. The seven scene keys are `scene_number`, +`timestamp_start`, `timestamp_end`, `visual`, `audio`, `shot_type` and `label`. +Keep source/asset identity, evidence locators, precision and gaps alongside the +scenes when needed. Choose useful labels for the actual content; advertising +categories should not force a tutorial or film into a marketing interpretation. + +Check chronological intervals against the actual duration and clock, including +the tail. Internally continuous times can still drift from the source. Cite the +actual frames/ranges supporting important claims. State partial coverage rather +than stretching an uninspected final scene. A completed provider job, more scenes, +valid JSON or distinct wording does not establish correct analysis. diff --git a/kimi/skills/visual-analysis/scripts/concat-images.mjs b/kimi/skills/visual-analysis/scripts/concat-images.mjs new file mode 100644 index 0000000..56be49b --- /dev/null +++ b/kimi/skills/visual-analysis/scripts/concat-images.mjs @@ -0,0 +1,305 @@ +#!/usr/bin/env node + +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { basename, dirname, extname, join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; + +const parsed = parseArgs(process.argv.slice(2)); +const outputPath = resolve(parsed.output); +const outputExtension = extname(outputPath).toLowerCase(); + +if (![".jpg", ".jpeg", ".png", ".webp"].includes(outputExtension)) { + fail("--output must end in .jpg, .jpeg, .png, or .webp"); +} +if (parsed.inputs.length === 0 || parsed.inputs.length > 25) { + fail("Pass between 1 and 25 input images"); +} +if (parsed.captions.length && parsed.captions.length !== parsed.inputs.length) { + fail("Pass exactly one --caption per input image, or omit captions entirely"); +} + +const inputs = parsed.inputs.map((input) => resolve(input)); +for (const input of inputs) { + if (!existsSync(input)) fail(`Input image does not exist: ${input}`); +} + +mkdirSync(dirname(outputPath), { recursive: true }); +const temporaryDirectory = mkdtempSync( + join(dirname(outputPath), ".concat-images-"), +); +const imageMagick = resolveImageMagick(); +// Some ImageMagick builds cannot discover installed fonts without fontconfig. +const captions = parsed.captions.length + ? parsed.captions + : inputs.map(defaultCaption); +const captionFont = [ + ...(captions.every((caption) => /^[\x00-\x7F]*$/.test(caption)) + ? [ + "/System/Library/Fonts/Menlo.ttc", + "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", + ] + : []), + "/System/Library/Fonts/Supplemental/Arial Unicode.ttf", + "/System/Library/Fonts/Supplemental/Arial.ttf", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + join(process.env.WINDIR ?? "C:\\Windows", "Fonts", "arial.ttf"), +].find((font) => existsSync(font)); + +try { + const prepared = inputs.map((input, index) => { + const preparedPath = join(temporaryDirectory, `${index}.png`); + const args = imageMagick.prefix.concat([ + input, + ...(parsed.crop ? ["-crop", parsed.crop, "+repage"] : []), + "-auto-orient", + "+profile", + "icc", + "-resize", + `${parsed.tileWidth}x${parsed.tileWidth}>`, + preparedPath, + ]); + run(imageMagick.command, args, `prepare ${input}`); + return preparedPath; + }); + + const dimensions = run( + imageMagick.command, + imageMagick.prefix.concat(prepared, ["-format", "%w %h\n", "info:"]), + "read image dimensions", + ) + .trim() + .split("\n") + .map((line) => line.split(" ").map(Number)); + const aspect = + Math.max(...dimensions.map(([width]) => width)) / + Math.max(...dimensions.map(([, height]) => height)); + const captionHeight = parsed.noCaptions ? 0 : 28; + let columns = parsed.columns ?? 1; + let tileWidth = 0; + for ( + let candidate = 1; + candidate <= Math.max(inputs.length, parsed.columns ?? 1); + candidate++ + ) { + if (parsed.columns && candidate !== parsed.columns) continue; + const rows = Math.ceil(inputs.length / candidate); + const width = Math.floor( + Math.min( + parsed.tileWidth, + 1568 / candidate - 2 * parsed.gap, + (Math.floor(1568 / rows) - 2 * parsed.gap - captionHeight) * aspect, + ), + ); + if (width > tileWidth) { + tileWidth = width; + columns = candidate; + } + } + if (tileWidth < 1) fail("The requested columns/gap leave no room for images"); + const tileHeight = Math.max(1, Math.floor(tileWidth / aspect)); + const captioned = prepared.map((input, index) => { + const output = join(temporaryDirectory, `captioned-${index}.png`); + run( + imageMagick.command, + imageMagick.prefix.concat([ + input, + "-resize", + `${tileWidth}x${tileHeight}`, + "-background", + "#000000", + "-gravity", + "center", + "-extent", + `${tileWidth}x${tileHeight}`, + ...(parsed.noCaptions + ? [] + : [ + "(", + "-size", + `${tileWidth}x${captionHeight}`, + "xc:#eeeeee", + ...(captionFont ? ["-font", captionFont] : []), + "-fill", + "#111111", + "-stroke", + "none", + "-pointsize", + "18", + "-gravity", + "west", + "-annotate", + "+5+0", + captions[index], + ")", + "-append", + ]), + output, + ]), + "add caption below image", + ); + return output; + }); + const montageArgs = imageMagick.montagePrefix.concat([ + ...(captionFont ? ["-font", captionFont] : []), + ...captioned, + "-tile", + `${columns}x`, + "-geometry", + `+${parsed.gap}+${parsed.gap}`, + "-background", + "#eeeeee", + outputPath, + ]); + run(imageMagick.command, montageArgs, "compose contact sheet"); + + process.stdout.write( + `${JSON.stringify( + { + columns, + crop: parsed.crop ?? null, + inputs: inputs.map((input, index) => ({ + caption: captions[index], + path: input, + })), + ...(parsed.noCaptions + ? {} + : { + note: "The light-gray strips below the images contain tool-added position labels, not video content.", + }), + width: columns * (tileWidth + 2 * parsed.gap), + height: + Math.ceil(inputs.length / columns) * + (tileHeight + captionHeight + 2 * parsed.gap), + outputPath, + rows: Math.ceil(inputs.length / columns), + }, + null, + 2, + )}\n`, + ); +} finally { + rmSync(temporaryDirectory, { force: true, recursive: true }); +} + +// Frame filenames carry the exact coordinate returned by the preview tools. +function defaultCaption(input) { + const name = basename(input, extname(input)); + const timeline = + name.match(/^timeline-(\d+)f-\d+ms-\d+$/) ?? + name.match(/^timeline-frame-(\d+)$/); + if (timeline) return `f${timeline[1]}`; + const source = + name.match(/^source-(\d+)ms-\d+$/) ?? name.match(/^asset-frame-(\d+)ms$/); + if (source) return `${(Number(source[1]) / 1000).toFixed(3)}s`; + return basename(input); +} + +function parseArgs(args) { + const result = { + noCaptions: false, + captions: [], + columns: undefined, + crop: undefined, + gap: 0, + inputs: [], + output: undefined, + tileWidth: 720, + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--no-captions") result.noCaptions = true; + else if (arg === "--caption") + result.captions.push(requiredValue(args, ++index, arg)); + else if (arg === "--columns") + result.columns = boundedInteger( + requiredValue(args, ++index, arg), + arg, + 1, + 9, + ); + else if (arg === "--crop") result.crop = requiredValue(args, ++index, arg); + else if (arg === "--gap") + result.gap = boundedInteger( + requiredValue(args, ++index, arg), + arg, + 0, + 100, + ); + else if (arg === "--output") + result.output = requiredValue(args, ++index, arg); + else if (arg === "--tile-width") + result.tileWidth = boundedInteger( + requiredValue(args, ++index, arg), + arg, + 100, + 2000, + ); + else if (arg === "--help" || arg === "-h") usage(0); + else if (arg.startsWith("-")) fail(`Unknown option: ${arg}`); + else result.inputs.push(arg); + } + if (!result.output) fail("--output is required"); + if (result.crop && !/^\d+x\d+\+\d+\+\d+$/.test(result.crop)) { + fail("--crop must use WIDTHxHEIGHT+X+Y pixel geometry"); + } + return result; +} + +function resolveImageMagick() { + if (commandExists("magick")) { + return { command: "magick", montagePrefix: ["montage"], prefix: [] }; + } + if (commandExists("convert") && commandExists("montage")) { + return { command: "convert", montagePrefix: [], prefix: [] }; + } + fail("ImageMagick is unavailable; expected magick or convert + montage"); +} + +function commandExists(command) { + return ( + spawnSync("sh", ["-c", `command -v ${command}`], { + stdio: "ignore", + }).status === 0 + ); +} + +function run(command, args, step) { + const executable = + step === "compose contact sheet" && command === "convert" + ? "montage" + : command; + const result = spawnSync(executable, args, { encoding: "utf8" }); + if (result.status !== 0) { + fail( + `${step} failed: ${(result.stderr || result.stdout || "unknown error").trim()}`, + ); + } + return result.stdout; +} + +function requiredValue(args, index, option) { + const value = args[index]; + if (!value) fail(`${option} requires a value`); + return value; +} + +function boundedInteger(value, option, minimum, maximum) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) { + fail(`${option} must be an integer from ${minimum} to ${maximum}`); + } + return parsed; +} + +function fail(message) { + process.stderr.write(`concat-images: ${message}\n`); + process.exit(1); +} + +function usage(exitCode) { + process.stdout.write( + "Usage: concat-images.mjs --output [--columns N] [--crop WxH+X+Y] [--caption text ...] \n", + ); + process.exit(exitCode); +} diff --git a/tests/visual-analysis.test.mjs b/tests/visual-analysis.test.mjs new file mode 100644 index 0000000..7895c7d --- /dev/null +++ b/tests/visual-analysis.test.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +const hosts = ["codex", "claude", "grok", "kimi"]; +const reference = "references/scene-breakdown.md"; +const read = (host, file) => + readFile( + new URL(`../${host}/skills/visual-analysis/${file}`, import.meta.url), + "utf8", + ); + +test("all plugin hosts ship one visual-analysis entrypoint and the same portable scene reference", async () => { + const expected = await read("codex", reference); + for (const host of hosts) { + const skill = await read(host, "SKILL.md"); + assert.match(skill, /^name: visual-analysis$/m); + assert.ok(skill.includes(`](${reference})`)); + assert.equal(await read(host, reference), expected); + await read(host, "scripts/concat-images.mjs"); + } +}); + +test("the portable JSON example preserves source coordinates and the seven scene fields", async () => { + const text = await read("codex", reference); + const examples = [...text.matchAll(/```json\s*([\s\S]*?)```/g)].map((m) => + JSON.parse(m[1]), + ); + assert.ok(examples.length); + for (const example of examples) { + assert.equal(example.time_domain, "source"); + for (const scene of example.scenes) { + assert.deepEqual(Object.keys(scene).sort(), [ + "audio", + "label", + "scene_number", + "shot_type", + "timestamp_end", + "timestamp_start", + "visual", + ]); + } + assert.equal(example.evidence.audio_basis, "chatcut_transcript"); + } +}); + +test("Grok and Kimi verification retain a working route to the shipped analysis skill", async () => { + for (const [host, entry] of [ + ["grok", "verification"], + ["kimi", "chatcut-verification-guide"], + ]) { + const text = await readFile( + new URL(`../${host}/skills/${entry}/SKILL.md`, import.meta.url), + "utf8", + ); + assert.ok(text.includes("](../visual-analysis/SKILL.md)")); + await read(host, "SKILL.md"); + } +});