panoptik-studio.vercel.app

{
  "name": "get_project_state",
  "kind": "answer",
  "impl": "imperative",
  "description": "MANDATORY FIRST STEP for any video editing, review, or enhancement request. Returns the full project: clips, durations, zooms, overlays, audio, plus the Director Playbook and the autonomous decision tree for vague requests like 'edit this video'. All timestamps are CURRENT TIMELINE seconds; check timelineRevision — if it changed since your last read, your cached observations are stale and you MUST re-ingest.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "list_clips",
  "kind": "answer",
  "impl": "imperative",
  "description": "Returns all timeline video clips with cumulative start/end timestamps, durations, speeds, and transitions. All times are CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "list_scenes",
  "kind": "answer",
  "impl": "imperative",
  "description": "Returns all timeline segments (scenes) with cumulative timeline start/end timestamps, duration, speed, and transitions. All times are CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "get_silence_intervals",
  "kind": "answer",
  "impl": "imperative",
  "description": "Detects audio silence and dead-air intervals where the speaker is inactive, returned in CURRENT TIMELINE seconds. Use for ripple trimming: dead air >= 1.5s is a cut candidate ({op:'cut', dropSilence:true}). Requires transcription — call generate_captions first if status is PENDING_TRANSCRIPTION.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "minDurationSec": {
        "type": "number",
        "description": "Minimum silence duration threshold in seconds (default 1.0)."
      }
    }
  },
  "page": "/"
}

{
  "name": "get_transcript",
  "kind": "answer",
  "impl": "imperative",
  "description": "Returns the spoken transcript with timestamps in CURRENT TIMELINE seconds. If empty, call generate_captions to transcribe. Re-fetch this after any cut/delete/speed change — earlier timestamps go stale when the timeline shifts.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "inspect_timeline",
  "kind": "act",
  "impl": "imperative",
  "description": "Detailed inspector of all clips, timeline timestamps, transitions, speed, zooms, and overlays. All times are CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "get_click_log",
  "kind": "answer",
  "impl": "imperative",
  "description": "Returns cursor telemetry in CURRENT TIMELINE seconds. Default response is COMPACT: per-20s attention buckets (where the mouse was active), click counts, and suggested zoom timestamps — plus exact cursor (x, y) interpolation via atTimestamp. Pass detail:'full' for the raw point list (token-heavy; rarely needed). Click clusters are prime zoom candidates; a parked cursor (no recent events) is NOT a good zoom target.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "atTimestamp": {
        "type": "number",
        "description": "Optional: interpolate and return the exact cursor (x, y) at this timeline timestamp"
      },
      "detail": {
        "type": "string",
        "enum": [
          "summary",
          "full"
        ],
        "description": "'summary' (default) returns attention buckets only; 'full' also includes the raw point list."
      }
    }
  },
  "page": "/"
}

{
  "name": "export_clip",
  "kind": "act",
  "impl": "imperative",
  "description": "Renders and exports the edited video locally in the browser via WebCodecs. Prompts the human for confirmation before encoding. Returns a downloadable Blob URL and file metadata upon completion.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "format": {
        "type": "string",
        "enum": [
          "mp4",
          "webm"
        ],
        "description": "Video container format. 'mp4' (H.264) is most compatible; 'webm' (VP9) is smaller."
      },
      "resolution": {
        "type": "string",
        "enum": [
          "720p",
          "1080p",
          "4k"
        ],
        "description": "Export resolution. '1080p' is standard HD, '720p' is faster, '4k' is ultra high-res."
      },
      "fps": {
        "type": "number",
        "enum": [
          24,
          30,
          60
        ],
        "description": "Output frame rate (default 30)."
      }
    },
    "required": [
      "format",
      "resolution"
    ]
  },
  "page": "/"
}

{
  "name": "propose_zoom_points",
  "kind": "act",
  "impl": "imperative",
  "description": "Proposes zoom-in keyframes at specific timeline timestamps. Proposals appear as ghost diamond markers on the timeline for human review — they are NOT applied until commit_staged_changes. Timestamps are CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "timestamps": {
        "type": "array",
        "items": {
          "type": "number"
        },
        "description": "Timeline seconds where zoom-in keyframes should be placed."
      },
      "scale": {
        "type": "number",
        "minimum": 1.2,
        "maximum": 5,
        "description": "Zoom magnification depth (default 2.2). Range 1.2 to 5.0."
      },
      "focalX": {
        "type": "number",
        "minimum": 0,
        "maximum": 1,
        "description": "Normalized focal X center (0 to 1, default 0.5)."
      },
      "focalY": {
        "type": "number",
        "minimum": 0,
        "maximum": 1,
        "description": "Normalized focal Y center (0 to 1, default 0.5)."
      }
    },
    "required": [
      "timestamps"
    ]
  },
  "page": "/"
}

{
  "name": "add_text_overlay",
  "kind": "act",
  "impl": "imperative",
  "description": "Stages a styled text caption or annotation overlay at a specific timeline timestamp and screen position. Supports custom fonts, sizes, colors, backdrops, and entrance/exit animations. Staged overlays appear for review until committed. Timestamps are CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "text": {
        "type": "string",
        "description": "The text content to display (no emojis)."
      },
      "timestamp": {
        "type": "number",
        "description": "When the text should appear (timeline seconds)."
      },
      "duration": {
        "type": "number",
        "description": "Duration in seconds (default 3.0)."
      },
      "position": {
        "type": "string",
        "enum": [
          "top",
          "bottom",
          "center"
        ],
        "description": "Vertical position on screen (default 'bottom')."
      },
      "fontSize": {
        "type": "number",
        "description": "Font size in pixels (e.g. 24, 36, 48)."
      },
      "fontFamily": {
        "type": "string",
        "description": "Font family (e.g. 'Inter', 'Outfit', 'Montserrat', 'Playfair Display', 'Fira Code')."
      },
      "fontWeight": {
        "type": "string",
        "enum": [
          "normal",
          "bold",
          "600",
          "800",
          "900"
        ],
        "description": "Font weight (e.g. 'bold', '600', '900')."
      },
      "fontStyle": {
        "type": "string",
        "enum": [
          "normal",
          "italic"
        ],
        "description": "Font style ('normal' or 'italic')."
      },
      "color": {
        "type": "string",
        "description": "Text color hex or rgba (e.g. '#ffffff', '#facc15')."
      },
      "backgroundColor": {
        "type": "string",
        "description": "Backdrop pill color (e.g. 'rgba(15,23,42,0.85)', '#1e293b')."
      },
      "borderRadius": {
        "type": "number",
        "description": "Backdrop corner radius in pixels (e.g. 8, 12)."
      },
      "animation": {
        "type": "string",
        "enum": [
          "none",
          "fade",
          "pop",
          "slide-up",
          "slide-down",
          "zoom-in",
          "typewriter",
          "bounce"
        ],
        "description": "Entrance/exit animation (default 'fade')."
      }
    },
    "required": [
      "text",
      "timestamp"
    ]
  },
  "page": "/"
}

{
  "name": "set_background",
  "kind": "act",
  "impl": "imperative",
  "description": "Stages a background color or gradient change. Fills the stage padding area around the video when aspect ratio padding is visible.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "kind": {
        "type": "string",
        "enum": [
          "solid",
          "gradient"
        ],
        "description": "Background type: 'solid' (single color) or 'gradient' (two-stop linear gradient)."
      },
      "color": {
        "type": "string",
        "description": "Hex color for solid background, e.g. '#0a0a0a'."
      },
      "stops": {
        "type": "array",
        "items": {
          "type": "string"
        },
        "description": "Array of two hex colors for gradient, e.g. ['#007cf0', '#7928ca']."
      }
    },
    "required": [
      "kind"
    ]
  },
  "page": "/"
}

{
  "name": "generate_captions",
  "kind": "act",
  "impl": "imperative",
  "description": "Transcribes the video's audio (camera/mic + screen tracks separately) and generates timestamped additive subtitle phrases across the timeline — same pipeline as the manual Auto-Generate button. Always call this first if get_transcript is empty — cuts, zooms and silence detection all depend on speech timestamps.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "language": {
        "type": "string",
        "description": "Audio language: 'auto', 'hinglish', 'en', 'hi', 'es', 'fr', 'de', 'ja', 'zh'. Default 'auto'."
      },
      "includeSpeakerLabels": {
        "type": "boolean",
        "description": "Prefix captions with 'Speaker:' / 'Screen:' (default true, matches the manual Speaker toggle)."
      },
      "presetId": {
        "type": "string",
        "description": "Caption style preset id: viral, clean, outline, subtitles, electric, highlighter (default 'viral')."
      },
      "speakerFontSize": {
        "type": "number",
        "description": "Font size px for Speaker captions (default 36)."
      },
      "screenFontSize": {
        "type": "number",
        "description": "Font size px for Screen captions (default 28)."
      }
    }
  },
  "page": "/"
}

{
  "name": "split_clip",
  "kind": "act",
  "impl": "imperative",
  "description": "Splits the clip at a timeline timestamp into two segments. Use before delete_clip to remove an unwanted range. Timestamps are CURRENT TIMELINE seconds. The response includes the new clip map — use it, not your pre-split observations.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "timestamp": {
        "type": "number",
        "description": "Timeline second where the split cut should occur."
      }
    },
    "required": [
      "timestamp"
    ]
  },
  "page": "/"
}

{
  "name": "split_segment",
  "kind": "act",
  "impl": "imperative",
  "description": "Splits the video clip at the given timeline timestamp. Timestamps are CURRENT TIMELINE seconds. The response includes the new clip map.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "timestamp": {
        "type": "number",
        "description": "Timeline second where the split should occur."
      }
    },
    "required": [
      "timestamp"
    ]
  },
  "page": "/"
}

{
  "name": "delete_clip",
  "kind": "transact",
  "impl": "imperative",
  "description": "Deletes a clip segment and ripple-joins the adjacent clips. THIS SHIFTS THE TIMELINE: every timestamp after the deleted range moves earlier. The response flags timelineShifted — you MUST re-ingest get_project_state and get_transcript afterwards and never reuse pre-delete timestamps. Cannot delete the only remaining clip.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "clipIndex": {
        "type": "number",
        "description": "0-based index of the clip segment to delete."
      },
      "segmentId": {
        "type": "string",
        "description": "Optional direct segment ID."
      }
    }
  },
  "page": "/"
}

{
  "name": "set_clip_transition",
  "kind": "act",
  "impl": "imperative",
  "description": "Sets the incoming transition effect and duration on a clip segment. Does not shift timestamps.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "clipIndex": {
        "type": "number",
        "description": "0-based index of the clip segment."
      },
      "transition": {
        "type": "string",
        "enum": [
          "cut",
          "fade",
          "dipToBlack",
          "slide-left",
          "slide-right",
          "zoom-in",
          "wipe"
        ],
        "description": "Transition style effect."
      },
      "duration": {
        "type": "number",
        "description": "Transition duration in seconds (default 0.45)."
      }
    },
    "required": [
      "clipIndex",
      "transition"
    ]
  },
  "page": "/"
}

{
  "name": "set_clip_speed",
  "kind": "act",
  "impl": "imperative",
  "description": "Sets playback speed for a clip segment (0.5x, 1x, 1.5x, 2x, 3x with pitch-preserved WSOLA audio). THIS SHIFTS THE TIMELINE: the segment's duration changes, so later timestamps compress or expand. Re-ingest after calling.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "clipIndex": {
        "type": "number",
        "description": "0-based index of the clip."
      },
      "speed": {
        "type": "number",
        "enum": [
          0.5,
          1,
          1.5,
          2,
          3
        ],
        "description": "Playback speed multiplier."
      }
    },
    "required": [
      "clipIndex",
      "speed"
    ]
  },
  "page": "/"
}

{
  "name": "set_speed",
  "kind": "act",
  "impl": "imperative",
  "description": "Sets playback speed multiplier for the selected segment (0.5x, 1x, 1.5x, 2x). THIS SHIFTS THE TIMELINE — re-ingest after calling.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "speed": {
        "type": "number",
        "enum": [
          0.5,
          1,
          1.5,
          2
        ],
        "description": "Playback speed multiplier."
      }
    },
    "required": [
      "speed"
    ]
  },
  "page": "/"
}

{
  "name": "set_aspect",
  "kind": "act",
  "impl": "imperative",
  "description": "Sets the stage aspect ratio preset ('16:9', '9:16', '1:1', '4:3', or 'source'). Does not shift timestamps.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "preset": {
        "type": "string",
        "enum": [
          "16:9",
          "9:16",
          "1:1",
          "4:3",
          "source"
        ],
        "description": "Aspect ratio preset."
      }
    },
    "required": [
      "preset"
    ]
  },
  "page": "/"
}

{
  "name": "add_music",
  "kind": "act",
  "impl": "imperative",
  "description": "Adds or moves an audio track on the timeline at a specified start timestamp (timeline seconds).",
  "inputSchema": {
    "type": "object",
    "properties": {
      "trackId": {
        "type": "string",
        "description": "ID of the audio track."
      },
      "startT": {
        "type": "number",
        "description": "Timeline start offset in seconds."
      }
    },
    "required": [
      "trackId",
      "startT"
    ]
  },
  "page": "/"
}

{
  "name": "get_director_guidelines",
  "kind": "answer",
  "impl": "imperative",
  "description": "Returns the authoritative Video Director reasoning playbook: multimodal heuristics, zoom scales, text overlay rules (no emojis), keepouts, the staleness contract, and the autonomous decision tree for vague requests like 'edit the reaction video'.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "get_video_summary",
  "kind": "answer",
  "impl": "imperative",
  "description": "Returns the complete semantic summary of the loaded video in CURRENT TIMELINE seconds: transcript phrases, scene breakdown, silence intervals, dead air, facecam corner, and the Director Playbook. Call this to understand the video before editing; call generate_captions first if the transcript is missing.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "get_scene_detail",
  "kind": "answer",
  "impl": "imperative",
  "description": "Lazy drill-down: returns raw click coordinates, bounding box, and word-level timestamps for a single scene. Windows are in CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "sceneId": {
        "type": "number",
        "description": "Scene index"
      }
    },
    "required": [
      "sceneId"
    ]
  },
  "page": "/"
}

{
  "name": "probe_frames",
  "kind": "act",
  "impl": "imperative",
  "description": "Samples video frames at timeline timestamps: returns deterministic feature summaries and base64 snapshot images with an optional A1..C3 alphanumeric 3x3 grid for visual grounding. Snapshot rendering can take seconds on long recordings — request only the timestamps you need and set includeSnapshot:false when you only need features. Use the grid cells to reference screen regions in locate_visual_target. Timestamps are CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "timestamps": {
        "type": "array",
        "items": {
          "type": "number"
        },
        "description": "List of timeline timestamps to query"
      },
      "includeSnapshot": {
        "type": "boolean",
        "description": "Whether to generate base64 image snapshots for visual inspection (default: true)"
      },
      "gridOverlay": {
        "type": "boolean",
        "description": "Whether to overlay a 3x3 alphanumeric grid (A1..C3) on the snapshot (default: true)"
      }
    },
    "required": [
      "timestamps"
    ]
  },
  "page": "/"
}

{
  "name": "locate_visual_target",
  "kind": "act",
  "impl": "imperative",
  "description": "Grounds a visual target into a safe zoom focal point via a 3-tier hierarchy: (1) click telemetry near the timestamp (confidence 1.0), (2) parse VLM output — Gemini-style 0-1000 bbox, 3x3 grid cell (A1..C3), or normalized x/y — from probe_frames snapshots, (3) fallback center with guidance. Timestamps are CURRENT TIMELINE seconds.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "query": {
        "type": "string",
        "description": "Target element description, e.g. 'elephant' or 'search bar'"
      },
      "timestamp": {
        "type": "number",
        "description": "Timeline timestamp to search in"
      },
      "scale": {
        "type": "number",
        "description": "Desired zoom depth (default: 2.2)"
      },
      "vlmOutput": {
        "type": "string",
        "description": "Raw VLM response containing bbox or grid cell (if already called)"
      }
    },
    "required": [
      "query",
      "timestamp"
    ]
  },
  "page": "/"
}

{
  "name": "propose_edits",
  "kind": "act",
  "impl": "imperative",
  "description": "The main editing tool: applies one atomic batch of edit ops in a single call. STAGE BASELINE (automatic): every batch applies the signature look — gradient backdrop, 28px stage padding, 16px rounded frame corners — unless the batch carries its own {op:'bg'}. ZOOMS PERSIST: zoom windows auto-extend until the next scene boundary or a sustained cursor move to a different region (minimum 4s hold). Ops (times in CURRENT TIMELINE seconds): {op:'cut', t0, t1} removes the exact window — the deterministic form, pass the intervals from get_silence_intervals; {op:'cut', t, dropSilence:true} expands t to the matching silence interval; a cut without a window or matching silence is REJECTED, never silently skipped. Also: {op:'zoom', t0, t1, cx?, cy?, scale?, ease?}; {op:'text', t, text, pos?:'top'|'bottom'|'center', dur?, fontSize?, fontWeight?, color?, backgroundColor?, animation?}; {op:'cam', corner:'tl'|'tr'|'bl'|'br', shape?, size?}; {op:'bg', kind:'solid'|'gradient', c0, c1?}; {op:'trans', at, kind, dur?}; {op:'speed', t0, t1, mult}. mode 'replace' clears prior agent zooms/overlays but NEVER Whisper captions. MANDATORY RULES: (1) NO emojis in text. (2) Multi-stage sequential pans (1.6x-1.8x) for long content. (3) Invert overlay position vs zoom half. (4) Verify targets via probe_frames, not a parked cursor. AFTER CUTS: the response sets cutsApplied and timelineShifted — re-ingest get_project_state + get_transcript before any further op batch. CHECK rejectedOps: a rejected cut means dead air was NOT removed.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "ops": {
        "type": "array",
        "description": "Array of EditOp objects (see description for the exact shapes)",
        "items": {
          "type": "object"
        }
      },
      "plan": {
        "type": "string",
        "description": "Concise explanation of your editorial strategy"
      },
      "mode": {
        "type": "string",
        "enum": [
          "replace",
          "append"
        ],
        "description": "replace (default) or append"
      }
    },
    "required": [
      "ops"
    ]
  },
  "page": "/"
}

{
  "name": "commit_staged_changes",
  "kind": "act",
  "impl": "imperative",
  "description": "Commits ALL staged ghost proposals (from propose_zoom_points, add_text_overlay, set_background, generate_captions) permanently. Shows the staged diff dialog for human confirmation. Edits already applied via propose_edits need no commit.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "discard_staged_changes",
  "kind": "act",
  "impl": "imperative",
  "description": "Clears all pending ghost edit proposals without modifying the committed project.",
  "inputSchema": {
    "type": "object",
    "properties": {}
  },
  "page": "/"
}

{
  "name": "cloud_transcribe",
  "kind": "act",
  "impl": "imperative",
  "description": "Transcribes project audio using Cloud Whisper Large v3 with word timestamps (Requires Pro or BYOK). Prefer generate_captions first — it both transcribes and stages caption overlays.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "language": {
        "type": "string",
        "description": "Optional ISO language code (e.g. 'en')"
      }
    }
  },
  "page": "/"
}

{
  "name": "ai_auto_director",
  "kind": "act",
  "impl": "imperative",
  "description": "Hosted 1-click auto-director: sends the video digest to a cloud model (Claude Haiku / Gemini Flash) which returns a full edit plan, applied via the same snapping pipeline as propose_edits (Requires Pro or BYOK). Air-gapped mode blocks it — use the local tool pipeline instead.",
  "inputSchema": {
    "type": "object",
    "properties": {
      "instruction": {
        "type": "string",
        "description": "Optional user creative guidance"
      }
    }
  },
  "page": "/"
}