// SCENE TEMPLATE. Copy to engine/scenes/<id>.js and add it to engine/timeline.js.
// Write the beat-by-beat plan here first: every lyric word + onset (engine/data/lyrics.json), the visual event, its layer
// (BACK / MID / FRONT), the camera move, and which beat or downbeat each cut lands on.
//
// PLAN
//   t      word/event          visual                                   layer   camera / cut
//   0.00   (hard cut in)       ...                                      ...     ...
//
// Layer recipe over footage ("type behind the subject"):
//   BACK graphics -> drawPlate(v) -> MID graphics -> drawPlate(v, {matte:true}) -> FRONT graphics (tracked to the subject)
// Pure-graphics moments skip the plates and render 2D/3D only.
import * as THREE from "three";

const CLIP = "clip_a"; // a folder in assets/video/ (prepped by tools/prep_clip.sh)
const SLOT = 0.0; // song time at clip frame 1, for clips generated against the song (lip-sync, beat-guided dance): local = t - SLOT
let v = null,
  g,
  scene,
  cam,
  word;

// smooth a tracked landmark over +-2 frames (raw pose jitter looks cheap)
function smoothPt(v, localT, idx) {
  let x = 0,
    y = 0,
    n = 0;
  for (let k = -2; k <= 2; k++) {
    const tr = v.trackAt(localT + k / v.meta.fps);
    const p = tr?.pose?.[idx];
    if (p && p[2] > 0.3) {
      x += p[0];
      y += p[1];
      n++;
    }
  }
  return n ? [x / n, y / n] : null;
}

export default {
  async load(E, layer) {
    try {
      v = await E.loadVideo(CLIP);
    } catch (e) {
      console.warn("footage missing", CLIP);
    }
    g = E.makeG2D(E.W, E.H); // one full-res 2D surface, reused for MID and FRONT
    // 3D hero type: emissive front face (HDR > 1 blooms on dark looks), darker extrusion sides
    scene = new THREE.Scene();
    cam = E.safeCamera(new THREE.PerspectiveCamera(40, E.W / E.H, 0.1, 200)); // origin = center of the safe band
    cam.position.set(0, 0, 10);
    word = new THREE.Mesh(
      E.text3D("HELLO", { font: "ArchivoBlack", size: 1.6, depth: 0.45 }),
      [
        new THREE.MeshBasicMaterial({ color: E.col(E.PAL.accent, 1.3) }),
        new THREE.MeshBasicMaterial({ color: E.col(E.PAL.ember, 0.8) }),
      ],
    );
    scene.add(word);
    // off landscape, pull back until the word spans 75% of the safe band (its pop overshoots about 1.3x at the peak)
    if (E.ORIENT !== "landscape") {
      word.geometry.computeBoundingBox();
      const bb = word.geometry.boundingBox,
        tanHalfW = Math.tan(THREE.MathUtils.degToRad(cam.fov / 2)) * cam.aspect;
      const fill = (0.75 * E.SAFE.w) / E.W;
      cam.position.z = Math.max(
        10,
        (bb.max.x - bb.min.x) / fill / (2 * tanHalfW),
      );
    }
  },
  async prepare(E, t) {
    if (v) await v.frame(Math.max(0, t - SLOT));
  }, // never let a clip freeze: keep localT inside the clip
  render(E, t, rt) {
    const S = E.SCALE,
      c = g.ctx,
      local = t - SLOT;
    E.renderer.setRenderTarget(rt);
    const w = E.words.find((x) => x.w.toUpperCase() === "HELLO");
    if (v) {
      // clean plate: 'cover' fills the frame when the clip's shape matches the canvas, 'contain' windows it when not
      const fit = E.plateFit(E, v);
      E.drawPlate(E, v, { fit });
      g.clear(); // MID: giant type behind the subject
      // Landscape keeps the full-bleed slam. Narrower canvases fit the word to the safe band and pop it in from below 1,
      // so the overshoot stays inside the band.
      const hero = { font: "Archivo", weight: 900, stretch: "125%" };
      const R = E.SAFE,
        wide = E.ORIENT === "landscape";
      const size = wide
        ? 420 * S
        : E.fitSize(c, "HELLO.", { ...hero, size: 420 * S }, R.w * 0.8);
      E.drawText(c, "HELLO.", R.x + R.w / 2, R.y + R.h * 0.45, {
        ...hero,
        size,
        color: E.PAL.accent,
        align: "center",
        s: w
          ? E.popScale(t, w.s, { k: 300, z: 0.45, from: wide ? 1.6 : 0.6 })
          : 1,
        alpha: w ? E.wordAlpha(t, w.s, w.e) : 1,
      });
      g.commit();
      E.blitG2D(E, g, 1);
      E.drawPlate(E, v, { fit, matte: true }); // subject back in front, same framing
      g.clear(); // FRONT: label locked to the nose (pose landmark 0)
      const p = smoothPt(v, local, 0);
      if (p) {
        const [x, y] = E.plateToScreen(E, v, p, { fit });
        c.strokeStyle = E.PAL.white;
        c.lineWidth = 3 * S;
        c.strokeRect(x - 60 * S, y - 70 * S, 120 * S, 140 * S);
        E.drawText(c, "SUBJECT_01", x + 70 * S, y - 60 * S, {
          font: "JetBrains Mono",
          weight: 700,
          size: 22 * S,
          color: E.PAL.white,
          align: "left",
        });
      }
      g.commit();
      E.blitG2D(E, g, 1);
    } else {
      word.scale.setScalar(Math.max(0.001, w ? E.popScale(t, w.s) : 1)); // pure-graphics fallback: 3D word on the onset
      E.renderer.render(scene, cam);
    }
    E.post.punch = 0.03 * E.pulse("kicks", t, 0.09); // camera punch on kicks (moves the frame, never alters pixels)
    E.hud.theme = "dark"; // 'light' on white/colour frames
    E.hud.sub = true; // false only while this scene shows the full sung line itself
  },
};

// Delivery canvas, chosen once in the brief: 'landscape' (1920x1080), 'portrait' (1080x1920), 'square' or 'WxH'.
// render.mjs/still.mjs --canvas overrides it to render the same project at a second ratio.
export const CANVAS = "landscape";

// Optional safe-band override per orientation, as fractions of the canvas; sides it does not name keep their default.
// A 9:16 canvas defaults to the platform-UI band (top 0.14, bottom 0.35, left 0.06, right 0.13), anything else to the full frame.
// export const SAFE = { landscape: { top: 0.1, bottom: 0.1, left: 0.1, right: 0.1 } };

// Master edit decision list. One entry per section scene (hard cuts at section boundaries, on downbeats or just
// before a vocal pickup), plus the global HUD on top. Scenes render only inside [start, end). id = file name in engine/scenes/.
//   file:    optional, use another file than <id>.js
//   variant: optional alternate version of a section (e.g. two endings). Only variant 'a' renders unless you pass --variant b.
//            Entries without a variant always render.
//   canvas:  optional 'portrait' | 'landscape' | 'square' (or a list): the entry renders only on that canvas. Use it when one
//            section needs a different layout per ratio; entries without it render on every canvas.
export const TIMELINE = [
  { id: "intro", file: "_template.js", start: 0.0, end: 16.66, z: 0 }, // replace with your own scene files
  // { id: 'hook1',  start: 16.66, end: 37.60, z: 0 },
  // { id: 'hook1_v', start: 16.66, end: 37.60, z: 0, canvas: 'portrait' }, // with canvas: 'landscape' on hook1
  // { id: 'outro',   start: 140.0, end: 150.0, z: 0, variant: 'a' },
  // { id: 'outro_b', start: 140.0, end: 153.0, z: 0, variant: 'b' },     // may run past the song: render.mjs pads silence
  { id: "hud", start: 0.0, end: 9999, z: 100 },
];

// Persistent frame. See engine/scenes/hud.js for every option.
export const HUD = {
  title: "SONG TITLE",
  mark: null, // or { path: '<svg path d>', w, h, alphas: [1] }
  sections: [
    [0, "INTRO"],
    [16.66, "HOOK_01"],
  ],
  counters: [
    // The spine lives here when the concept has one, e.g. a clock or a counter that grows over the song:
    // { label: '', keys: [[0, 120], [150, 420]], fmt: 'clock', suffix: ' AM' },
    // { label: 'COUNT', keys: [[0, 1], [40, 10000], [150, Infinity]], interp: 'log', fmt: 'short' },
  ],
  progress: true,
};

// Project-wide post defaults (merged over the neutral engine defaults; scenes can still override per frame via E.post).
// Neutral = footage passes through untouched. Use grade:'warm' only for amber-on-black, mostly graphics pieces.
export const POST = {
  // grade: 'warm', grain: 0.045, vignette: 0.35, ca: 0.0015, invertCol: '#FFA010',
};

# Scene-agent brief (template)

Copy this to `AGENTS_BRIEF.md` in the project and fill in the `<…>` parts. Delete lines that don't apply (for example the footage sections in an all-graphics video). Each scene agent gets a short prompt: its section, times, lyrics, must-have ideas and clips, and an instruction to read this brief first.

---

You are one of <N> motion designers. Each of you builds one section of a <duration> music video for "<song>" by <artist>. It is rendered entirely in JavaScript (three.js + Canvas2D) by a deterministic frame-stepping engine at 1080p60 on a <landscape 1920×1080 | portrait 1080×1920> canvas<, over AI-generated footage of <subject>>.<If both ratios ship: your scene renders on both canvases, or the lead gives you a portrait twin file.>

**The director's brief, in their words:** "<paste the key adjectives and asks: energy, pace, looks, what to avoid>". Treat this as your showreel.

## Read first (in this order)

1. `STYLE.md`: the law (looks, palette, type, motion grammar, footage rules, bans).
2. `TREATMENT.md`: your section, its lyrics and must-haves, and the footage map.
3. `ENGINE_API.md`: the full API.
4. `analysis/research_*.md`: the technique library with timings, and the verified facts and strings for inserts. Only use verified numbers.
5. Your clips: `ffmpeg -i assets/clips/<clip>.mp4 -vf fps=4,scale=320:-2,tile=6x4 -frames:v 1 out/<id>/sheet_<clip>.jpg`, then Read it. Also look at one matte (`assets/video/<clip>/m_00050.png`).
6. `engine/scenes/_template.js` (and the lead's `_smoke.js`, if present): a working example of plate → type behind the subject → matte → tracked label.

## Ownership

- You own ONLY `engine/scenes/<id>.js`, plus an optional `engine/scenes/<id>/` helper folder and `assets/<id>/` for generated data.
- Don't edit `engine/main.js`, `canvas.js`, `core.js`, `plate.js`, `typekit.js`, `roto.js`, `timeline.js`, `scenes/hud.js` or anyone else's files. If you hit an engine bug, work around it locally and report it.
- **Deterministic only:** no `Math.random`, `Date`, `performance.now` or rAF timing. Use `E.rng(seed)`, `E.hash1`, `E.noise1/2`.
- **Performance:** at most ~35 ms per frame at 1080p. Cache in `load()`, at most 2 full-res 2D canvases redrawn per frame, and no allocation in hot loops.
- Don't generate new AI footage or images unless the lead asks. Procedural textures in code are fine.

## Footage rules

- **Footage is shown clean**: `E.drawPlate` only. No filter, glow, grain, outline, halftone, tint or grade on the subject's pixels. You may reframe, crop, mirror, cut, speed-ramp, mask into shapes, tile, or put the plate on a 3D card.
- **Plate fit per canvas:** draw every plate with `fit: E.plateFit(E, v)` (same options in `plateToScreen`), so a clip whose shape differs from the canvas is windowed, never cover-cropped to fill.
- **Integration:** every footage moment needs at least one graphic that is tracked to the subject, timed to its motion, continues a line or shape in the plate, or sits behind it via the matte. A plain "video + caption" is a failure.
- **Never a frozen frame.** Keep `localT` inside the clip's duration. Slot clips (lip-sync, beat-guided dances) use `local = t − slot` and aren't shifted; other clips can use any offset.
- **Never cover the face** during lip-sync. MID-layer type behind the subject must stay readable, so move it if a key letter is hidden.
- No cut-off heads (except a deliberate face ECU), and give full-body shots room.
- Clips: <clip · slot · one-line content with key moments in clip seconds · matte quality>.

## Lyric rules

- **Every word visible.** Every sung word is on screen at its onset, either as a designed hero or in the HUD subtitle (on by default). Set `E.hud.sub=false` only while you show the full line.
- **Impact on the onset.** The impact frame is `w.s`. Anticipation starts at most 4 frames earlier, and nothing lands late.
- **Hold until at least `w.e`.** Exit 3× faster than the entry, or hard-cut on the next beat.
- **Hierarchy:** 1-3 hero words per line. Never use the same type treatment on two consecutive lines.

## Motion rules

- Springs with overshoot for pops, ease-out for arrivals, in-out for morphs, linear only for counters and progress. UI never scales from 0.
- Kicks drive camera punch or shape hits, snares drive color or shape events, vocals drive type, bass drives glow. Use only 1-2 sync layers per moment.
- Cut density follows the section's energy in TREATMENT.md. A designed event lands on every beat.
- At least one "how did they do that" moment per section: type behind the subject, a line that leaves the footage and becomes type, a 3D object that comes out of a 2D drawing, a match cut on shape, or a tile explosion.
- Set `E.hud.theme` for your look (`'light'` on white or color frames).
- **Lay out from `E.W`, `E.H` and `E.SAFE`,** never from 1920 or 1080 constants. On a portrait canvas, every word, logo and face stays inside `E.SAFE` (`motion_library.md` §10).

## Tools (run from the project root)

- **Stills and sheet:** `node tools/still.mjs --only <id>,hud --range <a>:<b>:<step> --scale 0.5 --dir out/<id>/it1 --sheet sheet --cols 4`, then Read the sheet. Use `--t 1.2,3.4` for exact times.
- **Preview with audio:** `node tools/render.mjs --only <id>,hud --from <a> --to <b> --fps 30 --scale 0.5 --out out/<id>/preview.mp4 --force`.
- **Perf:** `node tools/render.mjs --only <id> --from <a> --to <a+2> --fps 60 --scale 1 --out out/<id>/perf.mp4 --force`, and read the printed fps.
- **Freeze check:** `ffmpeg -i out/<id>/preview.mp4 -vf "freezedetect=n=0.003:d=0.25" -map 0:v -f null - 2>&1 | grep freeze_`.
- A process boot takes about 10 s, so batch many times into one still call. Other agents render at the same time, so stay at scale 0.5 except for final checks.

## Required verification loop

1. **Plan first.** Write a beat-by-beat plan as a comment block at the top of your file: each lyric word with its onset (from `engine/data/lyrics.json`), the visual event, the layer (back/mid/front), and where each cut lands (from `engine/data/audio.json`).
2. **Build, then contact-sheet** the whole section at 0.25 s steps and critique it.
3. **Onset check:** stills at `w.s + 0.03` for every hero word (legible and at its peak) and at `w.s − 0.10` (not fully in yet).
4. **Preview mp4** at 30 fps with audio. Confirm the cuts land on beat frames, and run the freeze check.
5. **Iterate at least 3 times on the look:**
   - Would a top motion designer repost this?
   - Is there one focal point?
   - Is the face clear and the footage untouched?
   - Is the type hierarchy strong?
   - Is it too empty, too busy, or generic?
6. **Boundaries:** the first frame is a hard cut in and must be striking. Check the first and last 3 frames.
7. **Safe band** (portrait canvas, or both ratios): a sheet with `--safe` (and `--canvas portrait` when the project's canvas is landscape). Nothing that must read sits outside the dashed band.

## Final report (to the lead)

- What you built, moment by moment (time → visual → layer), and which footage you used.
- Measured 1080p fps.
- Paths to your 4 best full-res stills (`--scale 1`).
- Engine issues, lyric-timing doubts and freeze-check results.

# Editing an approved soundtrack

Use this workflow when the director requests an audio change. Keep the original recording, current approved master, and candidates separate. Record the source asset or file hash, sample rate, channels, requested words, destination window, candidate source window, model parameters, selected take, and crossfade lengths in the project's edit notes.

## Choose the smallest useful edit

- **Pronunciation or one lyric:** audition a corrected performance, then replace only that phrase. Build the sheet sent to the model from the hand-corrected `analysis/lyrics.txt`, changing only the target word: a sheet that differs elsewhere makes the take sing other lines differently, which leaves only the target span usable. Repaint supplies a new take within a window; it does not guarantee the same singer, melody or backing. The `scenario-ace-step` skill explains when a high-strength Cover candidate is worth trying for voice continuity, and the Repaint recipe for changing sung words. Do not regenerate the whole soundtrack into the final cut merely to repair one line.
- **Different energy, genre or singer:** offer distinct short auditions before recutting the picture. A new recording needs a new timing analysis even if its requested BPM and lyrics match.
- **Trim, remove, repeat or rearrange existing material:** use deterministic editing. Prefer musical phrase boundaries and verify both joins; a crossfade alone does not fix a cut across a sung syllable or conflicting beats.

For generation, discover the named family with `search`, inspect the selected member with `model_schema_get`, and upload the source or reuse its asset id. Use the current schema and sibling skill for model-specific fields, not guessed parameters. Price each distinct payload with `model_run(dry_run=true)` inside the authorized budget, launch with `wait=false`, and use `jobs_wait` again on pending jobs. A timeout is not a reason to regenerate. Display candidates with `asset_display`; download with `asset_download` without an audio conversion `format`.

## Audition before replacing

Locate the phrase with the supplied lyrics, waveform and transcription together. Give the model enough context around consonants and breaths; record the replacement interval separately from that context. A returned file may be a full song or a section: inspect its duration and align the intended span in both recordings instead of assuming its origin is zero or that its timestamps match the master.

Present short original/candidate previews with identical surrounding context and clear take labels. For acronyms, a phonetic spelling can help the model sing separate letters; retain normal spelling in displayed lyrics. Compare pronunciation, singer identity, pitch, rhythm, accompaniment, loudness and transitions. Neither a transcript nor waveform similarity proves that it sounds right. If listening is unavailable, state that limitation and let the director select from previews. An existing selection authorizes applying that take without another confirmation once its span is located in both recordings; a selected take that lacks the corrected performance goes back to the director with the measurements, and the master stays. If no candidate works, keep the master and revise the attempt only within the authorized retry scope and budget. Unattended, with no prior take selection or explicit delegation to choose, stop after saving the labeled previews and list the pending choice; do not apply a take or spend on another batch to avoid waiting.

## Apply the selected span

Cut the span locally from the downloaded take's lossless decode by default: it is free, places both boundaries on any sample, and the cut lands where the splice happens. The fixed first-party audio trimmer `model_scenario-audio-cut` (the platform's one deterministic audio trimmer) suits a span that must stay on Scenario as an asset. Inspect its schema first: at authoring time its times moved in 0.1 s steps, and its download is the file as produced, a WAV with `outputFormat` wav. Round its window outward so the whole phrase survives, or upload the local cut with `upload_asset` when the clip must keep exact boundaries. As documented in `scenario-ace-step`, downloaded spans can be spliced locally because MCP has no audio-join tool; do not invent a join API or substitute another generative run.

Decode the original once to a lossless working master. Match the candidate's sample rate and channel layout, align it to the destination, and preserve the original sample count for an in-place correction. Do not stretch the whole song to fit a patch. Start with short crossfades, for example 10-30 ms, adjust by audition, and keep every blended sample inside the declared edit window. Match gain locally rather than normalizing the whole master. Compare decoded samples outside the window for exact equality before lossy export. A deliberately length-changing edit instead needs an explicit timeline mapping and new downstream timings.

## Reconnect and verify

Save a uniquely named master and point `engine/data/audio.json.master` at it. Use a new basename for analysis: `audio_analysis.py` reuses stems already present under that name. Transcription, plots and vocal slices must use the stem for the master named in `engine/data/audio.json`, resolved by `audio_paths.py`; a missing stem is an error, never a reason to use an older take. Refresh the vocal stems and envelopes, rerun transcription before alignment, and regenerate affected vocal slices and lip-sync shots. For an in-place correction, keep the approved beat events and word timings outside the edit windows and take the new values only inside them: the analysis tools rewrite every event, and in one test moved a downbeat 0.5 s and words up to 2 s on unchanged audio. If timing or structure changed, rerun analysis and update cuts, beat accents and lip-sync footage.

Check sample count, clipping, boundary clicks, gaps and local loudness, and audition each join in context. Render a preview with the actual captions and cuts, then run `av_sync_check.py` against the approved master. Exact sample preservation is a lossless-master claim, not a claim about an AAC/MP3 export. For picture-only revisions, stream-copy the approved encoded audio when container compatibility and duration allow; otherwise encode from the approved master once and check timing. Report what was measured, what was auditioned, and any remaining uncertainty separately.

# Engine API (assets/engine → project engine/)

A deterministic three.js + Canvas2D compositor. Every pixel is a pure function of global song time `t`. `tools/render.mjs` and `tools/still.mjs` step frames with `window.renderFrame(frame, fps)` in headless Chrome, and ffmpeg muxes the untouched master audio on top.

## Contents

- [Files](#files)
- [Scene contract](#scene-contract)
- [Canvas and safe band](#canvas-and-safe-band)
- [The E object](#the-e-object)
- [Footage: clean plates, mattes and tracking](#footage-clean-plates-mattes-and-tracking)
- [Keyed footage planes in 3D (roto)](#keyed-footage-planes-in-3d-roto)
- [Post](#post)
- [HUD](#hud)
- [Timeline, variants and project config](#timeline-variants-and-project-config)
- [Fonts and 3D assets](#fonts-and-3d-assets)
- [Commands](#commands)

## Files

| file                                  | role                                                                                                                                                             |
| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `index.html`                          | fonts (@font-face → assets/fonts), importmap for three                                                                                                           |
| `main.js`                             | loads data and scenes, per-frame layer compositing, bloom, final grade shader, `renderFrame` / `grab`                                                            |
| `canvas.js`                           | canvas presets, orientation, the platform-UI safe band, per-canvas timeline filtering                                                                            |
| `core.js`                             | math, easing, springs, RNG/noise, palette, audio/lyric timeline helpers, text canvases, 3D type, G2D surfaces, video frame sequences (frames + matte + tracking) |
| `plate.js`                            | `drawPlate`, `plateRect`, `plateToScreen`, `plateMaterialFor`: clean footage with optional matte cut-out                                                         |
| `typekit.js`                          | kinetic type: `popScale`, `wordAlpha`, `drawText`, `fitSize`, `drawStagger`, `drawKaraoke`, `drawSubtitle`, `blitG2D`, `stretchKW`                               |
| `roto.js`                             | `makeRotoMaterial`: luma-keyed footage planes for clips shot on a black void (stylized redraws only with `{stylized:true}`)                                      |
| `timeline.js`                         | `CANVAS`, optional `SAFE`, `TIMELINE` (scene ids, [start,end), z, variant, canvas), `HUD` config, `POST` project defaults                                        |
| `scenes/hud.js`                       | global HUD layer (z=100): corners, title + mark, section code, bar/beat, counters (the spine), progress, subtitle track                                          |
| `data/audio.json`, `data/lyrics.json` | written by `audio_analysis.py` / `lyrics_align.py`                                                                                                               |

## Scene contract

```js
export default {
  async load(E, layer) {}, // build scenes/geometry/materials, load videos (cache everything here)
  async prepare(E, t, layer) {}, // optional: await v.frame(t - SLOT) for every clip drawn this frame
  render(E, t, rt, layer) {
    // t = GLOBAL seconds; window is [layer.start, layer.end)
    E.renderer.setRenderTarget(rt); // rt is pre-cleared transparent; renderer.autoClear is false
    // draw back → plate → mid → matte → front; E.renderer.clearDepth() between 3D passes
    // may set E.post.* and E.hud.* ; may return a number = layer opacity
  },
};
```

- Layers composite premultiplied-normal over black in z order (an entry can set `blend: 'add'|'screen'|'multiply'`). HDR linear color. Values above the bloom threshold (1.05) bloom.
- Previews run at `E.SCALE = 0.5` (E.W = 960 on landscape, 540 on portrait). Always size 2D pixels as `px * E.SCALE`, and place them from `E.W`, `E.H` and `E.SAFE`, never from 1920 or 1080 constants. The camera aspect is `E.W/E.H`.
- Deterministic only. No `Math.random`, `Date`, `performance.now` or rAF timing. Use `E.rng(seed)`, `E.hash1`, `E.noise1/2`. Particles use closed-form positions (no accumulated state).
- The budget is about 35 ms/frame at 1080p. Cache text textures, geometry and canvases in `load()`. At most 2 full-res 2D canvases redrawn per frame, and no allocation in hot loops.
- The first frame at `layer.start` is a hard cut in, so make it striking. The last frames land on an accent.

## Canvas and safe band

One project renders at one canvas, `CANVAS` in timeline.js: `'landscape'` (1920×1080, the default), `'portrait'` (1080×1920), `'square'` (1080×1080) or `'WxH'` with even sides. `--canvas` on `render.mjs` and `still.mjs` overrides it to render the same project at a second ratio; an unreadable value stops the render with an error.

- `E.CANVAS` is `[w, h]` at scale 1, and `E.ORIENT` is `'landscape' | 'portrait' | 'square'`.
- `E.SAFE` is `{x, y, w, h}` in px: the band platform UI never covers. On a 9:16-tall canvas it clears roughly the top 14%, the bottom 35%, 6% on the left and 13% on the right (authoring-time values); on anything wider it is the whole frame, so landscape layouts are unchanged. `SAFE` in timeline.js overrides sides per orientation, e.g. `{ landscape: { top: 0.1, bottom: 0.1 } }` or `{ portrait: { top: 0.2 } }`; a side it does not name keeps its default.
- Words, logos and faces go inside `E.SAFE`; footage and full-bleed graphics may fill the canvas behind it. The HUD and the house subtitle already lay out inside it, and off landscape the subtitle shrinks a long line to fit (a line much over 45 characters gets small, so split it in the lyrics).
- `E.safeCamera(cam)` shifts a PerspectiveCamera's frame so the origin lands on the band's center (a no-op on a full-frame band). Use it for 3D heroes, then set the distance so the object fits the band's width (see the scene template).
- `E.fitSize(ctx, text, o, maxW)` returns the largest size, up to `o.size`, at which the text fits `maxW`. Hero type sized for a 1920-wide frame overflows a 1080-wide one; fit it to `E.SAFE.w` divided by the largest scale its move reaches (a word popping in from 1.6× fits `E.SAFE.w / 1.6`), or enter from below 1.
- `E.plateFit(E, v)` returns `'cover'` when the clip's shape matches the canvas and `'contain'` (a window) when it does not. Pass it as every plate's `fit` in a scene that renders on more than one canvas.
- `node tools/still.mjs --safe` outlines the band in magenta. The HUD draws it (even at `alpha:0`), so the `hud` entry must be in `TIMELINE`. Never pass `--safe` to a delivery render.

## The E object

- **Time and audio**
  - `E.t`, `E.frame`, `E.fps`, `E.dur`, `E.bpm`, `E.spb`.
  - `E.env(name, t)` for `mix|vocal|kick|snare|hats|bass` (≈0..1.5), plus `E.envAvg(name, t, win)`.
  - `E.last(list, t)` / `E.next(list, t)` return `{i, t, dt}` for the lists `beats|downbeats|kicks|snares|vocal_onsets`. There is no `hats` list (hats are an envelope only).
  - `E.pulse(list, t, decay=0.12, lead=0)` is an exponential decay since the last event (the workhorse).
  - `E.beat(t)` returns `{i, t0, t1, phase, bar, barT0, inBar, isDown, len}`.
- **Lyrics**
  - `E.words` holds `{w, s, e, li, wi, section, line}`.
  - `E.lineAt(t, lead)`, `E.wordAt(t, lead)`, `E.wordsIn(t0, t1)`, `E.section(name)`.
- **Math**
  - `E.clamp`, `E.lerp`, `E.remap`, `E.fract`.
  - `E.ease.{linear,inQuad,outQuad,inOutQuad,inCubic,outCubic,inOutCubic,inQuart,outQuart,inOutQuart,inExpo,outExpo,inOutExpo,outBack,inBack,outElastic,smooth}`.
  - `E.tween(t, t0, dur, fn)`, `E.spring(dt, k, z)` (0→1 damped step response).
- **Palette**
  - `E.PAL`: replace the values in `core.js` per project (from STYLE.md). Keep the role keys `black ink white accent`; the HUD and templates use them.
  - `E.col(hex, mul)` gives a linear THREE.Color; `mul > 1` makes it glow.
- **2D**
  - `E.makeG2D(w, h)` returns `{ctx, tex, clear(), commit()}`.
  - `E.blitG2D(E, g, intensity, blending)` draws the surface fullscreen into the current target.
- **Type**
  - `E.drawText(ctx, text, x, y, {font, size, weight, stretch:'125%'|'extra-condensed'…, tracking(em), color, alpha, align, s|sx|sy, rot, skewX, stroke, strokeOnly, glow, italic})`.
  - `E.measure(ctx, text, o)`.
  - `E.drawStagger(ctx, text, x, y, t, s, {stagger, dy, spring, e})`.
  - `E.drawKaraoke(ctx, E, line, t, {x, y, size, align, font, litColor, dimColor, maxW})`: `maxW` shrinks the whole line to fit.
  - `E.popScale(t, s, {k, z, from})` and `E.wordAlpha(t, s, e, {out})`; `E.LEAD = 0.05`.
  - `E.textTexture(text, o)` returns a cached `{tex, w, h, aspect}` for planes.
  - `E.text3D(str, {font:'ArchivoBlack'|'Anton'|'BlackHanSans', size, depth, bevel…})` returns a cached, centered TextGeometry. Use a material array `[front, side]`.
  - Canvas `fontStretch` only accepts keywords. `stretchKW` maps percentages.
  - Never stroke a variable font: overlapping contours show their inner lines. Use static fonts (Archivo Black, Anton) for outline type.

## Footage: clean plates, mattes and tracking

This is the default compositing mode. Footage is drawn exactly as generated, and code graphics live in layers around it.

- `const v = await E.loadVideo('clip')` loads `assets/video/<clip>/f_#####.jpg` + `meta.json {fps,n,w,h,lumLo,lumHi,warp?,mask?,track?}`. If `meta.mask` is set, the matte sequence `m_#####.png` loads too (`v.mtex`, white = subject). If `meta.track` is set, `track.json` loads too.
- In `prepare()`: `await v.frame(localT)`. Use `localT = t - SLOT` for slot clips. This updates `v.tex`, `v.mtex` and `v.i`. Beat-warp in `meta.warp` is applied automatically. The index clamps to the last frame, so **keep localT inside the clip**, because a held last frame reads as a freeze.
- `E.drawPlate(E, v, { fit:'cover'|'contain', zoom:1, x:0, y:0, rot:0, crop:[x,y,w,h] (image UV, y down), mirror:false, opacity:1, matte:false, invert:false, feather:0.12 })` draws into the current render target.
  - `matte:true` draws only the subject (alpha from the matte). `invert:true` draws only the background.
  - `x` and `y` are px offsets at 1080p scale (multiplied by `E.SCALE` internally).
  - `fit:'cover'` fills the canvas and crops the clip when the ratios differ; `fit:'contain'` shows the whole clip as a band or window. Use `fit: E.plateFit(E, v)` so a 16:9 clip on a portrait canvas is windowed, never cover-cropped to fill (`pipeline.md` §9).
- **Layer recipe** ("type behind the subject"): `drawPlate(v)`, then MID graphics (a `blitG2D` or a 3D render), then `drawPlate(v, {matte:true})`, then FRONT graphics.
- **Backdrop replacement** for plates shot on a flat color cyc: draw a color field, then `drawPlate(v, {matte:true})`. The backdrop becomes your color and the subject is untouched (the floor shadow is lost). Run `fixmatte.py` on those clips first.
- `E.plateRect(E, v, opts)` gives `{x, y, w, h, cx, cy, rot}` in screen px. `E.plateToScreen(E, v, [u, v], opts)` gives `[x, y]` in screen px (E.W × E.H, y down) for a video-UV point. Pass the same opts you used for `drawPlate`.
- **Tracking:** `v.trackAt(localT)` returns `{ pose: [[x,y,vis]×33] | null, box: [x0,y0,x1,y1], c: [cx,cy], a: area, top: [x,y] }`, all in video UV (0..1, y down).
  - MediaPipe indices: 0 nose, 2/5 eyes, 9/10 mouth corners, 11/12 shoulders, 13/14 elbows, 15/16 wrists, 19/20 index fingers, 23/24 hips, 27/28 ankles.
  - Pose is single-person and may lock onto the wrong person in a group shot. `box`, `c` and `top` come from the matte and cover everyone.
  - Pose needs a human-like body. For creatures, objects or products, use `box`, `c` and `top`.
  - Smooth landmarks over ±2 frames (see `smoothPt` in the scene template). Raw jitter looks cheap.
- `E.plateMaterialFor(v, {matte})` gives a material for your own mesh: footage on a 3D card, inside a tile grid, or through a type mask.

## Keyed footage planes in 3D (roto)

For clips shot on a pitch-black void that should float inside a 3D world without a matte:

- `const m = E.makeRotoMaterial(E, {footGain, keyLo, keyHi})`, then `E.rotoLevels(m, v)`. Each frame: `m.uniforms.tVideo.value = v.tex`.
- Other uniforms: `uCrop` (Vector4 x,y,w,h in video UV), `uMirror`, `uOpacity`, `uReveal`/`uRevealDir`, `uDissolve`.
- Put it on `new THREE.Mesh(new THREE.PlaneGeometry((v.meta.w / v.meta.h) * h, h), m)` (the clip's own aspect), or call `E.drawRotoFullscreen(E, m, {scale, x, y})`.
- Output is `rgb = c·m, alpha = key·m`, so dim rim halos add light. Near-black costumes go see-through over bright graphics. Fix this with a dark backing cut to the silhouette, or a tighter key (`keyLo 0.006, keyHi 0.03`).
- Stylized redraws (dots, edges, ASCII, scanline, cel) need `{stylized:true}`. Use them only when the director asks; effects on footage are rejected by default.

## Post

- `E.post` is reset every frame to the neutral defaults, merged with the project's `POST` from timeline.js:
  `{bloom:0.55, bloomRadius:0.35, bloomThreshold:1.05, exposure:1, ca:0, grain:0, vignette:0, flash:0, flashCol:[r,g,b], invert:0, punch:0, shake:[x,y], rot:0, scan:0, glitch:0, letterbox:0, sat:1, tint:[r,g,b], fade:1, grade:'neutral', invertCol:'#FFA010'}`
- **`grade:'neutral'`** (default): values ≤ 1 pass through unchanged (no tonemap, no clamp), so footage and any palette render true. Only HDR > 1 is compressed, hue-preserving with a white-hot core (neon reads like a tube). `invert` is a true RGB invert.
- **`grade:'warm'`**: for amber-on-black pieces that are mostly graphics. It clamps g ≤ r and b ≤ g (kills cool CA fringes), applies a 50% hue-preserving ACES blend and pulls saturated yellows toward amber. `invert` becomes a poster in `invertCol`. It shifts footage colors, so don't use it when footage must stay clean.
- `grain`, `ca`, `vignette`, `scan`, `glitch`, `sat` and `tint` touch every pixel, including footage. Keep them at 0 over footage unless the director wants that texture.
- `punch` (zoom), `shake` and `rot` move the whole frame. `shake` is in UV units (0.004 is a hard hit), not pixels: pixel values blow the frame up and show as corrupt plates. They work as camera accents over footage. `rot` samples clamp-to-edge, so add about 0.1 punch while tilted.
- `flash` is added in linear space before sRGB encoding, so 0.1 already reads strong. It is zeroed below 0.02.
- `letterbox` masks to a 2.39:1 window across the full width, which is a thin strip on a portrait canvas. There, frame a cinema moment as a boxed plate instead.

## HUD

A persistent frame glues 2D, 3D and footage sections into one piece, and its subtitle track keeps every lyric readable.

- Per frame: `E.hud = {alpha:1, sub:true, corners:true, label:'', theme:'dark'|'light', ink, accent}`. Use `theme:'light'` on white or color frames (dark ink). Set `sub=false` only while the scene shows the full sung line itself.
- Content comes from `HUD` in timeline.js: `title`, optional `mark` (an SVG path, official brand paths only), `sections`, `counters` (`fmt: int|short|sci|clock|text`, `interp: lin|log`), `progress`, and `dark`/`light` theme colors. The counters are where the video's spine usually shows: a clock, a countdown, or a number that explodes over the song.
- It lays out inside `E.SAFE`, so on a portrait canvas it frames the safe band rather than the screen edges.
- Retheme it per project or drop it (`alpha:0`). Some concepts want no HUD at all.

## Timeline, variants and project config

- `TIMELINE` entries: `{id, file?, start, end, z, blend?, variant?, canvas?}`. One file per section, with hard cuts at boundaries on downbeats.
- **Per-canvas entries:** `canvas:'portrait'` (or a list of orientations) renders the entry only on that canvas, `--only` included; tags are case-insensitive, an unknown one stops the render with an error, and so does an `--only` that leaves nothing to render. Use it for a section that needs its own layout per ratio: tag the landscape file `canvas:'landscape'` and its portrait twin `canvas:'portrait'`. Entries without `canvas` render everywhere.
- **Variants:** give alternate versions of a section (an ending, a hook) `variant:'a'|'b'|'c'`. Only `a` renders unless you pass `--variant b`, which lets you deliver several cuts from one project. A variant may run past the song; `render.mjs` pads silence (`apad`).
- `CANVAS` and `SAFE` set the canvas and its safe band (see [Canvas and safe band](#canvas-and-safe-band)). `POST` in timeline.js sets project-wide post defaults. `HUD` configures the frame.

## Fonts and 3D assets

- Canvas fonts loaded by `index.html`/`main.js`: `Archivo` (variable: wght 100-900, stretch 62-125% via `stretch:'125%'`), `Archivo Black`, `Anton`, `JetBrains Mono`, `Doto`, `Black Han Sans`, `Unbounded`, `Instrument Serif` (+ italic). To add a project font, drop the TTF in `assets/fonts/`, add an `@font-face` in `index.html` and a `document.fonts.load` line in `main.js`.
- 3D type fonts: `E.text3D(str, {font:'ArchivoBlack'|'Anton'|'BlackHanSans'})`. Add more with `core.loadFont3D(name, url)` in `main.js`.
- `import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'` and `SVGLoader` are available. Assets are served from the project root (`/assets/3d/*.glb`, `/assets/brand/*.svg`).

## Commands

- `node tools/still.mjs --only <id>,hud --range a:b:step --scale 0.5 --dir out/<id>/it1 --sheet sheet --cols 4` (or `--t 1.2,3.4`; add `--safe` to outline the safe band, `--canvas portrait` for the other ratio)
- `node tools/render.mjs --only <id>,hud --from a --to b --fps 30 --scale 0.5 --out out/<id>/preview.mp4 --force`
- Full: `node tools/render.mjs --fps 60 --scale 1 --workers 3 --crf 17 --out out/final/master.mp4 [--variant b] [--to <end>] [--canvas portrait]`

# Lessons learned

Every item here cost real time or credits once. Read the matching section before each phase. "Director" means whoever commissions the video. Their taste overrides these defaults, but these defaults are the right place to start.

## Contents

- [Direction defaults that held](#direction-defaults-that-held)
- [Song and lyrics](#song-and-lyrics)
- [Generation and moderation](#generation-and-moderation)
- [Lip-sync](#lip-sync)
- [Mattes, tracking and compositing](#mattes-tracking-and-compositing)
- [Engine and rendering](#engine-and-rendering)
- [Color](#color)
- [Agents](#agents)
- [Review and revisions](#review-and-revisions)
- [Delivery](#delivery)

## Direction defaults that held

- **Generated footage is shown clean.** Directors rejected effects on the footage itself: LED dots, halftone, scanlines, ASCII, posterize/cel, outline traces, glow, grain and color grades. To get a different look on the subject, _generate_ it in that look (a style frame, then animate it). Don't filter it.
- **Code goes on top of the footage and is integrated with it** (behind via the matte, on it via tracking, in front). Footage pasted onto a graphic background, or a split screen, was rejected.
- **Motion graphics first.** The video should feel like a motion designer's showreel: fast, kinetic type, 2D and 3D mixed. It still needs the subject in many sections, including full-body long shots, not just close-ups.
- **Type frames the subject, never covers the face.** That means vertical banner type on the empty third, a dim zone behind the subject, or hero type behind via the matte. Behind-the-subject type still has to read; a hidden first letter got flagged.
- **No static frames, ever.** A frozen plate (a clip that ran out, a slot clip held before its slot, a still image standing in for video) reads as broken. Every footage moment plays live.
- **Variety of looks per section** (dark neon, white editorial, flat color, B&W film, pixel), cut hard on downbeats. Directors who want lots of color need to give color references; without them the palette drifts toward safe choices.
- **The first 3 seconds decide whether people keep watching.** Open on the hook with the subject already performing, and make sure it works as a muted autoplay.
- **Brand assets are exact.** Use the official logo files in their official colors (never recolored to the palette), and only where the director says. A small "made with …" credit on the end card is welcome.
- **Showcase moments get time.** A logo build or end card played too fast to see until it ran at 0.35-0.5×. A pixel-art moment with many fast cuts was "too much", and two long shots worked better. It's fine for the video to run past the song with silence on an extended end card.
- **Offer variants instead of arguing.** Alternate endings rendered as timeline variants let the director pick (one picked the third of three).
- **Replace rather than patch.** A close-up the director dislikes gets a different shot, not a re-grade. A weak generated insert (a glowing node network) gets rebuilt in code as vector graphics.

## Song and lyrics

- Separate the vocals with Demucs first, and align on the vocal stem.
- **Whisper large-v3 word timestamps beat wav2vec/MMS forced alignment on singing.** MMS ran about 200 ms late. Do two passes, plain and with vocabulary hints, then hand-correct the text. Invented and technical words are often misheard.
- **Held notes pull the next short word early.** `lyrics_align.py` fixes this, except for legitimately long tokens (acronyms, numbers); pass those to `--no-fix`.
- **Check timing visually** with `plot_lyrics.py`. Word starts sit on the rise of a harmonic stack or a consonant burst.
- A chant Whisper hallucinates can be timed from vocal-envelope peaks (`--chant section:WORD`). Keep chant events on the half-beat grid; it reads right even when a syllable is slightly off.
- **Measure the tempo; don't trust labels.** A song generated "at 130 BPM" measured 98 with double-time drums. Use the event lists (`beats`, `kicks`…) and never a hard-coded BPM.
- Vocals often lead the downbeat with pickups, so check the first word of each section. Section cuts go on the downbeat or just before the pickup.
- The demucs stems, librosa and ffmpeg decode the mp3 consistently. The verified end-to-end offset was 0.0 ms.
- **Song models mangle accented names.** A French first name with an accent was sung as "her vape". Spell the name phonetically in the sheet sent to the model (the sound, in plain English letters), keep the real spelling in `lyrics.txt`, and read the transcript for every name before spending on footage. The same run asked for about 2:00 and got 3:04, then 2:14 after the sheet was shortened and an "about 2 minutes, very short intro" cue added: the sheet length and a duration cue steer the result, so check `ffprobe` before building anything on it.
- Match the audio edit to the request: a new musical direction calls for new auditions; a local correction calls for a candidate patch in a defined window. Do not treat a rejected splice as a ban on editing. Asking for a half-time bridge turned one take into a ballad.

## Generation and moderation

- **Style frames before video.** Generating a still per shot in its look (from the subject sheet) and then animating it gives better looks and consistency than text-to-video. It also lets the director approve the look cheaply.
- **Never send the song** to a video model. Moderation rejects it ("InputAudioSensitiveContentDetected"), and `generateAudio:true` reproducing the vocal fails the job. Use `generateAudio:false`, synthesized beat tracks for dances, and the isolated vocal stem for lip-sync (that was accepted).
- Reference-image mode (sheet + crop) held identity well at 720p. Clips come back at 24 fps, about 0.04 s longer than requested.
- **Sets that composite:** a pitch-black void with a hard rim light, a flat color cyc, or white high-key. A glossy floor came back as a gray gradient. Flat color cycs let you replace the backdrop via the matte.
- Beat alignment from a beat-track reference is loose (about 1-1.5σ above chance). Always beat-warp dance clips; it matched 14/14 motion peaks within 160 ms.
- Text baked into generated frames is never usable. Prompt "no text", and do all type in code.
- Budget anchors at 720p, relative to one style frame (confirm live prices with `dry_run`; 1080p ran about 2.5 times as much per clip): a 6 s clip costs about twenty, 10 s about thirty-five, a lip-sync correction pass about thirteen. A full video with about 17 shots plus frames and lip-syncs came to about five hundred.

## Lip-sync

- **Audio-to-video** from a style frame and a vocal slice gave the lip-sync shots the director liked most, at 1080p, with a camera motion option.
- The alternative is reference-audio video with the vocal stem, then a **lip-sync correction** model (cut-off sync mode), replacing frames in place under the same clip name.
- Measure it: `lipsync_check.py` correlates mouth openness (MediaPipe FaceMesh) with vocal RMS. About 0.5 is good, and 0.1-0.2 is weak. Tracking fails on wide shots or when a hand crosses the face. Held vowels read as "closed", so judge by eye too. Never shift a slot clip to chase a measured lag. Under 0.2, retake it in another look if the budget allows, or run a lip-sync correction pass on it (the `video2video` route in `pipeline.md` §4, about thirteen style frames); otherwise keep the shot wider or shorter so the mouth is not the focal point.
- The audio-to-video member capped audio at 10 s at authoring time (an 11 s slice failed with a 400, and so did a slice of exactly 10.0 s), so cut slices of 9.9 s at most; clips come back a few frames shorter than the slice.
- Cut vocal slices as WAV, not mp3 (priming delay). MediaPipe must be pinned to `0.10.14` (newer versions dropped `mp.solutions`).

## Mattes, tracking and compositing

- **Apple Vision foreground masks** (`tools/matte`) are free, fast and good on people, but they miss hair against white and fine detail such as fingers on keys. Inspect mattes over a bright color before designing around them.
- On flat color cycs, union a chroma key with the Vision matte (`fixmatte.py`); use `--white` on white sets.
- On pixel art, soft mattes look dirty. Use a hard binary mask on the pixel grid plus a 1-cell ink outline.
- Where a cut-out is poor, don't ship it: show the full plate inside a window or shape, or regenerate the shot in a style that keys cleanly.
- Pose tracking is single-person and may lock onto a backup dancer, and it fails on non-human subjects. The matte bbox, centroid and top point work for everything. Smooth landmarks over ±2 frames.
- Keyed (luma) footage: output is `rgb = c·m, alpha = key·m`, so dim rim halos add light and crop edges inside the halo show as boxes. Near-black costumes go see-through over bright type; back them with a dark silhouette.
- Reframe every shot for its final use: headroom, no cut-off heads (except a deliberate face ECU), and room for full-body shots.

## Engine and rendering

- Headless Chrome with `--use-angle=metal` runs WebGL on the GPU on macOS. A simple 1080p60 scene renders at about 28 fps per worker, and 3 workers in parallel are fine. A 2:00 video renders in about 5 minutes.
- Frame sequences decoded with `createImageBitmap` ignore WebGL `flipY`. Use `imageOrientation:'flipY'` with `tex.flipY=false` (already in core.js).
- **MSAA render targets only resolve inside `renderer.render()`**, so a scene that drew nothing showed its previous frame. main.js renders an empty scene into every layer after the scene renders.
- Canvas `fontStretch` only accepts keywords (`stretchKW` maps them). `drawKaraoke` over a left-aligned layout must force `align:'center'` per word.
- Bloom: a threshold just above 1.0 with strength about 0.5. Lower thresholds wash whole frames white.
- `start×fps` may not be an integer, so the first frame of a scene can belong to the previous one.
- ffmpeg `drawtext` contact sheets failed silently on this machine; `sheet.py` uses PIL.

## Color

- With footage on screen, keep the grade neutral (`grade:'neutral'`): values ≤ 1 pass through untouched, and only HDR graphics compress.
- For amber-on-black, graphics-led pieces, `grade:'warm'` solves known problems:
  - ACES turns #FFB000 lemon-yellow, which the hue-preserving blend and yellow pull fix.
  - Chromatic aberration creates cool fringes on white type, which the g ≤ r, b ≤ g clamp kills.
  - A pink cast on peach skin is fixed by re-clamping b ≤ 0.85·g.
- Fades to black through a flash go gray or muddy. Fade through color instead (white → brand color → dark brand color → black), or iris to pure #000. #0B0B0B "ink" is not black, and it showed as a raised black.

## Agents

- One agent per song section, each owning one scene file, all working in parallel. It stays coherent when they share STYLE.md, a written treatment with a footage map, ENGINE_API.md, research notes and an explicit verification loop.
- Build the engine, data, style bible, smoke scene and footage pipeline **before** launching agents. Message the agents as new clips land.
- For revisions, resume the same agents via SendMessage (they keep their context) with one shared notes file. Keep the agent ids in `analysis/agents.md`.
- A global audit message to all agents ("find and fix any static frame in your section") works well. Ask each for a measured check (freezedetect, stills) in its report.
- Watch for clip double-use across sections. Agents under load report low fps; the in-page frame time is the real number.

## Review and revisions

- Watch the whole cut as 0.5 s contact sheets (8×5 tiles per 20 s) and check each section boundary at −1/0/+1 frames.
- Director notes arrive as timestamps ("at 0:36 the cut-out is bad", "at 0:48 it's too static"). Map each to its owning section and agent, fix it, re-render that range, and show a still or short clip of the fix.
- Count the iterations, song and video separately. Directors want that number for the making-of.

## Delivery

- 1080p60, 2-pass x264 at about 21 Mb/s (maxrate 25M) with AAC 192k gives about 414 MB for 2:37, under X's 512 MB limit. Standard X accounts cap video at 2:20, so cut a teaser too.
- Always run `av_sync_check.py` on the delivered file: expect 0.0 ms audio and 0 to +1 frame visual. A lyric-led cut with a correlation under about 0.15 can read -2 or -3 frames because type leads by `E.LEAD`; confirm with onset stills.
- macOS filenames are case-insensitive: an encode named like its master with different case overwrote its own input.
- Don't delete a render output until you've confirmed it's incomplete. A stopped job may have finished the concat and only been encoding.

# Motion-design library

This is a menu of looks, type treatments, moves and timing numbers that held up in production. Pick from it to fit the song; don't run through it like a checklist. Frame counts are @60 fps. The tempo-relative units (beat, 8th, bar) come from the song's own `audio.json`; never hard-code a BPM. `E.spring(dt,k,z)` and `E.ease.*` are in `engine_api.md`.

## Contents

1. [Looks](#1-looks)
2. [Type system](#2-type-system)
3. [Motion-type vocabulary](#3-motion-type-vocabulary)
4. [Technique library (40 moves)](#4-technique-library-40-moves)
5. [Integrating graphics with footage](#5-integrating-graphics-with-footage)
6. [Transitions](#6-transitions)
7. [Timing, easing and springs](#7-timing-easing-and-springs)
8. [Sync and cut density](#8-sync-and-cut-density)
9. [Safety and default bans](#9-safety-and-default-bans)
10. [Portrait canvas](#10-portrait-canvas)

## 1. Looks

Most strong videos use **2-4 looks**, cut hard on downbeats, with each section owned by one look. A look is a ground, an ink, an accent and a rendering rule, plus a matching footage style for any generated shots. These archetypes have been tested; invent others.

| look                  | ground                | ink                                           | accent               | rendering rule                                                          | footage style that matches                                       |
| --------------------- | --------------------- | --------------------------------------------- | -------------------- | ----------------------------------------------------------------------- | ---------------------------------------------------------------- |
| **NIGHT / neon**      | pure black            | glowing warm lines and type (HDR 2-4 → bloom) | warm-white cores     | glow allowed; ≥70% black                                                | rim-lit subject on a black void; light-line drawings             |
| **PAPER / editorial** | white or bone         | ink black                                     | ONE accent           | flat, crisp vector edges, Swiss grid, hairline rules                    | high-key white set; ink or brush drawings                        |
| **POP / color field** | flat saturated field  | black or white                                | a second field color | flat, no glow; hard 0-blur drop shadows OK                              | subject on a matching color cyc (backdrop replaceable via matte) |
| **FILM / mono**       | black-and-white grain | white type                                    | none                 | soft serif and subtitle type                                            | 35mm B&W lip-sync                                                |
| **RISO / print**      | paper                 | 2-3 spot inks with misregistration            | n/a                  | halftone and overprint on graphics only                                 | riso-style generated stills animated                             |
| **PIXEL / 8-bit**     | black or flat color   | pixel type (Doto)                             | 1-2                  | integer pixel grid, no AA, hard binary sprite mask + 1-cell ink outline | pixel-art character generated and animated                       |
| **WIRE / blueprint**  | dark blue or black    | 1 px lines                                    | callout color        | wireframe 3D, dimension callouts                                        | wireframe or point renders                                       |
| **CLAY / toy**        | soft color            | dark ink                                      | n/a                  | soft shadows, rounded 3D                                                | clay-toy subject                                                 |

- **High-contrast pop pairs:** tangerine+cobalt, lemon+black, pink+cobalt, mint+black, cobalt+lemon.
- **Glow belongs to dark looks only.** Flat looks stay flat.
- **Color comes from references.** A director who wants "lots of color" needs a palette reference (images or hex) up front. Without one the model drifts back to safe palettes, which happened once.
- **Spread the subject's looks too.** A subject can appear in several renderings over one video (3D cartoon mostly, a 2D flat section, a photoreal film section, a pixel ending), each generated as its own style frame. It reads as range, as long as identity holds.

## 2. Type system

One family in many widths beats many families. Defaults that worked:

| role        | font                                               | treatment                                                                                                                                          |
| ----------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- |
| **HERO**    | `Archivo` variable (wght 100-900, stretch 62-125%) | UPPERCASE, tracking −0.03em, ultra-wide 125% or ultra-condensed 62% (never "normal" for heroes), 25-90% of frame height, accent full stop ("RUN.") |
| **3D HERO** | `E.text3D` Archivo Black / Anton                   | extruded and beveled. Dark looks: emissive front + dark side. Flat looks: unlit color front + black side. The camera flies through or orbits.      |
| **SYSTEM**  | `JetBrains Mono` 500-700                           | UPPERCASE, +0.06-0.12em, 14-28 px: terminals, logs, labels, callouts                                                                               |
| **LED**     | `Doto` 900                                         | numerals, counters, split-flap, scoreboards                                                                                                        |
| **SOFT**    | `Instrument Serif` italic                          | lowercase, big and soft, only for tender lines                                                                                                     |

- **Outline type uses static fonts only.** A stroked variable font shows its overlapping inner contours.
- **Hierarchy:** 1-3 hero words per line (nouns, verbs, numbers). Function words stay small.
- **Vary scale and density:** full-frame poster slams, mid-size kinetic phrases and quiet lower-third subtitles.

## 3. Motion-type vocabulary

Never use the same treatment on two consecutive lines.

- **width wave:** stretch animated per letter in a sine wave
- **weight ladder:** the same word stacked 100→900
- **smear slam:** a 2-3 frame horizontal stretch into place
- **outline echo wall:** repeated outline copies fill the frame, one solid
- **karaoke fill:** an accent fill sweeps through the letters in sync with the vocal
- **text on a path:** along a curve in the footage (hair, an arm, a road), an orbit or a ring
- **decode scramble:** mono glyphs resolve into the word
- **split-flap / odometer counters**
- **type as a window:** the plate seen through giant letters
- **3D extruded fly-through**
- **kinetic crop:** one giant letter fills the frame, then pull back to reveal the word
- **stagger cascade**
- **sticker pop:** white-bordered, rotated, bouncy
- **UI chat bubble / terminal typing** with a block cursor
- **word pile-up:** lines stack until the frame is full, then cut to black
- **vertical banner** on the empty third
- **neon ignition:** the outline stroke-draws with a flicker, then blooms
- **Swiss grid slam:** words snap into grid cells after the hairline rules draw

## 4. Technique library (40 moves)

"8th", "beat" and "bar" are the song's own grid. Every move lists its timing and a JS sketch.

**Type moves**

1. **Smear Slam.** A word hits center from scale 1.6 with 3 trailing ghosts (offset −6/−12/−18% W, alpha .35/.2/.1) that collapse into it. 8 fr outExpo, ghosts gone by fr 6, 1 fr flash `post.flash=0.08`. _Use:_ the first word of a hook line, or chant words.
2. **Directional Blur Slide.** Slides in from off-frame with motion blur made of 8 stacked draws at 1/8 alpha, spaced by `v*dt`. 6 fr outQuart. _Use:_ call-and-response words.
3. **Letter Rise Stagger.** Letters rise from a baseline clip mask. 14 fr per letter outQuint, stagger 2 fr (1 fr for fast words). _JS:_ `ctx.rect(0,base-cap*1.1,W,cap*1.1); ctx.clip(); y=base+cap*(1-e)`.
4. **Wave Drop.** Letters fall in with ±15° rotation and land on a spring k=600 z=0.35, stagger 1.5 fr. _Use:_ "falling" or "dropping" lyrics.
5. **Variable-Width Wave.** A traveling wave through `wdth` 62→125, period 1 beat, phase 0.12 beat per letter. Pre-rasterize width steps in `load()`. _Use:_ "breathe", "stretch", held notes.
6. **Echo Wall.** The center row is solid and 4-6 outline rows scroll in alternating directions at ±0.35 W/s. Add a row every 8th in a chant.
7. **Punctuation Portal.** The final "." (or an i-dot or a cursor) scales to fill the screen and becomes the next background. 6 fr inExpo, while the word whips away 4 fr earlier.
8. **Color-Field Flip.** Swap background and foreground on each kick with a hard cut, plus a 2 fr scale punch 1.04→1. `bg = FIELDS[E.last('kicks',t).i % n]`.
9. **Swiss Grid Slam.** A 12-column grid: hairline rules draw first (10 fr linear), then words drop into cells (8 fr outExpo, from scale 0.94) at the onset minus 3 fr.
10. **Split-Flap Counter.** Digits flip every 2 fr through hashed randoms and settle right to left, 4 fr apart, on outBack. _Use:_ times, years, counts, countdowns.
11. **Scramble Resolve.** Glyph noise resolves left to right over 20-30 fr, each char locking at `i*1.5 fr`, from a themed glyph pool ("01ACGT#%").
12. **Text Selection + Cursor.** A UI highlight box grows from the left (10 fr outQuart), its handles pop 3 fr later, and a block cursor blinks per beat.
13. **Type-on-Path Ring.** Mono caps orbit a circle at 0.1 turn per beat, snapping +0.05 turn on each kick.
14. **Letters Bigger Than Frame.** Single glyphs at 150% W, one per 8th, with a 12 fr linear push 1.0→1.08. The counters become windows.
15. **Weight Ladder.** The same word in 8 rows from Thin to Black, one step per 8th, with an arrow stepping down.

**Shape and 2D moves**

16. **Dot-Collapse → Nested Iris.** Collapse to a dot (12 fr inQuart), hold 1 fr, then two discs burst out, the second lagging 3 fr. _Use:_ section changes.
17. **Spring Morph + Ghost Outlines.** A shape morphs polygon→polygon each beat (k=267 z=0.30), with 4 lagging outline echoes and orbiting satellites. Resample to 128 pts and recompute the echoes from `t − k/60`.
18. **Truchet Wave Field.** A tile field of quarter-arcs that waves rotate 90°. 1 wave per bar, rising to 1 per beat in a build.
19. **Gooey Pills.** UI chips joined by metaball necks (SDF `smin`), with flicker and scramble, merging into one blob.
20. **Single-Line Draw.** One continuous pen line draws and morphs scribble → object → object at about 0.8 H/s, with a 24 fr inOutCubic morph and a leading pen dot. _Use:_ "sketch", "draw", "line".
21. **Line-Icon Draw-On.** 2 px SVG icons stroke on over footage (12 fr linear dash offset), then multiply 1→3→6.
22. **Tumbling Square → Object Pop.** A square rolls in with squash on landing, then pops into an object (k=400 z=0.5) with sub-parts lagging 3 fr.
23. **Liquid Splat Wipe.** A noise-perturbed SDF blob splats (12 fr outExpo), flings droplets and thins into a ring that reveals the next color.
24. **Falling Drop Beat-Marker.** A drop bounces three times, each on a beat. A calm breath moment.
25. **Confetti Burst.** 40-80 primitives with closed-form drag and gravity, spin ±360°/s, fading over 70 fr. _Use:_ celebrations and the final lock-up.
26. **Starburst + Orbit Text.** A 12-spoke rounded burst whose spokes grow staggered, with a ring and orbiting mono text, plus a kick pulse.
27. **Easing-Race Chart.** Balls race A→B with onion-skin ghosts over graph lines, lasting exactly 1 bar. _Use:_ "rising", "falling", comparisons.
28. **Halftone Dissolve.** A dot grid whose radius is driven by wipe progress, used to transition from footage into graphics. 20 fr inOutQuart. It touches the transition, never the subject at rest.
29. **Kaleidoscope / Mirror Tile.** Fold UV into 4→6→8→12 wedges on downbeats. Graphics only.

**3D moves (three.js)**

30. **Cube-Grid Ripple.** An InstancedMesh cube field (about 576 instances). Each kick spawns a radial ring that pushes the cubes in Z.
31. **Point-Cloud Morph.** `THREE.Points` morph grid → torus → globe → text points, 1 beat each on inOutQuart.
32. **Extruded Type Flythrough.** `E.text3D` words in a Z corridor. The camera travels one word per beat, linear, with a 6 fr outExpo surge on downbeats.
33. **Orbit Data Rings.** 3-5 tilted rings of ticks and mono text at different speeds. One ring snaps +30° on each onset.
34. **Isometric Block City.** Iso boxes rise in a diagonal stagger (outBack s=2.6). _Use:_ "build", "world", "city".
35. **Vortex Zoom-Through.** Nested rotating squares; zoom through on inOutExpo, landing on the downbeat. Loops seamlessly.
36. **Infinite Zoom (Droste) through type.** Zoom into a letter's counter that contains the scene, 1 bar per level, linear in log space.

**Camera and post moves**

37. **Kick Scale-Punch.** `E.post.punch = 0.035*E.pulse('kicks',t,0.09)`, plus small shake on snares.
38. **Whip Pan with Smear Frame.** Exit 5 fr inQuart, 1 smear frame (a stretched copy), then entry 6 fr outQuart, centered on the kick.
39. **HUD Frame.** Corner brackets, title, section code, bar and beat squares, a spine counter and a progress line, for the whole video (`scenes/hud.js`).
40. **Chant Metronome Grid.** Each chant syllable fills the next cell of a 4×4 grid on its onset, a kick inverts a cell, and the last word wipes the grid with a progress bar.

## 5. Integrating graphics with footage

The rule that made footage sections land: **code goes on top of the video, deeply integrated, never video pasted over graphics.** Every footage shot needs at least one graphic that reacts to it:

- **BACK** (under the plate): fields, grids and giant type. With a color-cyc plate, replace the backdrop around the subject.
- **MID** (plate → graphics → matte cut-out): type behind the subject but over the background. Keep enough of each hero word visible to read it; if the subject covers a key letter, move the word.
- **FRONT** (over everything, often tracked): a line leaves a fingertip, a label locks to the head, words land on a jump apex, a box tracks the face, type rides a curve in the plate.
- **Continue the plate:** a line or shape that exists in the footage (a horizon, a screen, a hand gesture) leaves it and becomes graphics.
- **Match cuts on shape:** cursor → sun, dot → planet, line → horizon, a logo glyph → an iris wipe.
- **Plates in windows:** footage inside type, split panels, a tile wall, or a 3D card. This is compositing, not an effect.
- Pure-graphics stretches (often 30-40% of the runtime) must be as strong as the footage moments, and the subject should return often.

## 6. Transitions

All on downbeats: color-field wipe, shape iris (circle, rhombus or the video's own glyph), type-as-window zoom, grid tile flip, whip with shape smears, match cut on shape, split-screen reveal, dot collapse and burst, punctuation portal, liquid splat.

## 7. Timing, easing and springs

**Durations by role (@60)**

| role                   | enter                              | hold (min) | exit                 | easing                   |
| ---------------------- | ---------------------------------- | ---------- | -------------------- | ------------------------ |
| hard-hitting word slam | 6-8 fr                             | 8 fr       | 4-5 fr               | outExpo in / inQuart out |
| normal word or line    | 12-16 fr                           | ≥ 20 fr    | 8-10 fr              | outQuart / inCubic       |
| letter stagger         | 12-14 fr per letter, offset 1-2 fr | -          | reverse, offset 1 fr | outQuint                 |
| UI card or chip        | 12 fr from scale 0.94              | -          | 8 fr to 0.96 + fade  | outQuart / inQuad        |
| morph in place         | 18-30 fr (≤ 1 beat)                | -          | -                    | inOutQuart / spring      |
| section transition     | 12-24 fr, landing ON the downbeat  | -          | -                    | inOutExpo                |
| counter or progress    | real duration                      | -          | -                    | linear                   |
| ambient drift          | loop ≥ 1 bar                       | -          | -                    | linear or sine           |

**Overshoot recipes**

| feel                    | recipe                   | overshoot | peak    |
| ----------------------- | ------------------------ | --------- | ------- |
| crisp UI pop            | outBack s=1.70158        | 10%       | -       |
| cartoon pop             | outBack s=2.6            | 20%       | -       |
| firm                    | `E.spring` k=1600 z=0.70 | 4.6%      | 6.6 fr  |
| snappy                  | k=900 z=0.60             | 9.5%      | 7.9 fr  |
| firm editorial          | k=300 z=0.45             | ≈20%      | ≈12 fr  |
| bouncy (engine default) | k=260 z=0.42             | 23%       | 12.9 fr |
| jelly morph             | k=267 z=0.30             | 37%       | 12.1 fr |
| quick jelly             | k=600 z=0.35             | 31%       | 8.2 fr  |
| wobbly                  | k=150 z=0.25             | 44%       | 15.9 fr |

Settle time is about 4/(z·√k) s.

**Rules**

- Arrivals use ease-out (expo or quart), morphs use in-out, counters and progress use linear, and exits and anticipation use ease-in.
- Never scale UI from 0: start at 0.94-0.98. Big type may come from ≥ 1.6× or from 0 with a smear.
- **Anticipation:** 3-4 fr of counter-motion at 8-12% of the move's distance.
- **Lead:** visual hits land 2-3 fr before the audio event (`E.LEAD = 0.05` s). The eye needs the frame before the ear.
- **Stagger:** fast words 1 fr per letter, normal 2 fr, UI lists 3-4 fr per item. A cascade finishes within 1 beat.
- **Exits** take 60-70% of the entry duration.
- **Holds:** a word needs ≥ 20 fr to be read (≥ 12 fr if it repeats), and a line needs ≥ 1 beat after its last word.
- **One hero per frame:** during a slam, secondary elements freeze or dim to 40%.
- **Camera (3D):** one dominant move per shot: dolly through type, orbit reveal or crash zoom. Use FOV 18-24° for poster compression and 70-95° for fly-throughs.

## 8. Sync and cut density

- **Pick 1-2 sync layers per moment.** Kick → scale punch or a shape hit. Snare → a shape or color event, or a 1-2 frame glitch. Vocal onset → type. Bass envelope → glow. Hats envelope → shimmer.
- **Cut discipline:** section changes on `downbeats`, word cuts on `vocal_onsets`, color flips and punches on `kicks`, shakes on `snares`.
- **Cut density should climb with energy.** A shape that worked: intro and verse a cut every 2 bars → pre-chorus 1 bar, then 1 beat in its last bar → drop 1 bar with beat events inside → build 1 beat, then a half-beat → chant a half-beat with 16th flashes. Let the bridge breathe with a long held shot.
- **Showcase moments need time.** A brand or logo reveal, a character turn or a sprite moment reads at 0.35-0.5× speed with few cuts. Directors rejected fast cut rates on these twice.
- **A spine:** let the edit carry the song's meaning, with a counter, clock or countdown that progresses through the video while the cut rate or the looks escalate with it.

## 9. Safety and default bans

- **Photosensitivity:** at most 3 full-frame luminance flips per second. Color-to-color flips of similar luminance can run faster; keep white↔black flips to ≤ 2/s.
- **Banned unless the director asks:** any filter, glow, grain, outline, halftone or color change on generated footage; lens flares; generic HUD clutter with no meaning; "AI slop" nebula or particle soup; glossy Pixar-style 3D graphics; cyberpunk cyan/magenta; real people's faces; logos other than the client's own (and those only where specified, in their official colors).

## 10. Portrait canvas

Layout rules for a 9:16 canvas (`CANVAS = 'portrait'`), written when the engine gained one. Unlike the rest of this file, they have been through only one production so far.

- **The frame is narrow and tall.** A hero line breaks into stacks of one or two words, each sized with `E.fitSize` to `E.SAFE.w` divided by the largest scale its move reaches, or entering from below 1 so the overshoot stays inside the band. A wide poster slam becomes a word pile-up down the band.
- **Short sung lines.** The house subtitle shrinks a line to fit the band, so a line much over 45 characters turns small: split it in `analysis/lyrics.txt`.
- **Type sits above the subject, not beside it.** The vertical banner on the empty third becomes a horizontal band above the head. Below the chest works only while it stays above the bottom 35%, where the platform caption sits.
- **Words, logos and faces stay inside `E.SAFE`.** Plates, color fields and full-bleed graphics fill the whole canvas behind them, so the platform UI covers background, never the read.
- **Moves run along the long axis:** slides, smear slams and wipes travel up and down.
- **3D heroes:** wrap the camera in `E.safeCamera(cam)` and fit the object to the band's width, as the scene template does.
- **Cinema moments:** `letterbox` is a thin strip on portrait. Use a boxed plate (`fit:'contain'`) with type above and below instead.
- **The first 3 seconds:** the hook lands big and centered in the band, and reads with the sound off, since many viewers scroll muted.

# Pipeline: commands and Scenario calls

## Contents

1. [Scaffold](#1-scaffold)
2. [Audio and lyrics](#2-audio-and-lyrics)
3. [Scenario: subject, style frames, assets](#3-scenario-subject-style-frames-assets)
4. [Scenario: footage routes](#4-scenario-footage-routes)
5. [Footage prep: frames, mattes, tracking, checks](#5-footage-prep-frames-mattes-tracking-checks)
6. [Engine and agents](#6-engine-and-agents)
7. [Assembly, grade and verification](#7-assembly-grade-and-verification)
8. [Delivery](#8-delivery)
9. [Two ratios from one project](#9-two-ratios-from-one-project)

## 1. Scaffold

```bash
bash <skill >/scripts/scaffold.sh /path/to/song.mp3 # run inside the new project folder
```

This creates `audio/master.*`, the `engine/` template, `tools/` (with the matte binary on macOS), fonts, npm deps (three, playwright) and a Python 3.11 `.venv`. On a machine that already has a project, you can symlink `.venv` and `node_modules` from it to save time.

## 2. Audio and lyrics

```bash
.venv/bin/python tools/audio_analysis.py audio/master.mp3                              # stems + engine/data/audio.json (prints BPM and downbeats)
.venv/bin/python tools/lyrics_align.py transcribe                                      # plain pass, also kept as analysis/lyrics_raw_plain.json
.venv/bin/python tools/lyrics_align.py transcribe --prompt "<names, jargon, acronyms>" # hinted pass (lyrics_raw_hints.json); align reads the latest
# → hand-write analysis/lyrics.txt (sections + corrected lines; the user's lyrics win if supplied)
.venv/bin/python tools/lyrics_align.py align analysis/lyrics.txt --no-fix <ACRONYMS >[--chant <section >: <WORD >]
.venv/bin/python tools/plot_lyrics.py 1:12 20:30 ... # Read the PNGs; fix outliers by hand in lyrics.json
```

- **Verify the tempo.** Beat trackers and song-generation prompts can be off by a factor (a "130 BPM" track measured 98 with double-time drums). Check the median beat dt against the kick envelope, and write the measured grid into TREATMENT.md.
- Treat lines you couldn't make out as guesses, and list them for the user at the end.
- **A repeated shout** (`--chant <section>:<WORD>`): write that section's words into `lyrics.txt` as sung. The aligner places one WORD per vocal peak between the lines around the section, at most as many as you wrote; an empty section takes every peak up to the next line, or to the song's end.

## 3. Scenario: subject, style frames, assets

- **Workspace.** Get the team and project with `teams_list`. If there is more than one, ask the user. Pass `team_id` and `project_id` on every call. Log every asset and job id in `analysis/jobs.md` as you go, because revisions need them.
- **Upload references.** Use the multipart flow: `upload_asset(file_name, content_type, kind, file_size)` returns a presigned URL. `curl -fsS -X PUT -T file '<url>'` (no checksum headers), then `upload_asset_complete(upload_id)`. Inline base64 is only for files under 100 KB.
- **Subject sheet first.** A subject can be a character, mascot, creature, band avatar or product.
  - Turn the user's reference images into a model sheet (turnaround, expressions, detail callouts) before any footage. Pick an image model that takes reference images (`recommend` with `capability: "img2img"`, then `model_schema_get`): pass the references through the model's reference-image field and ask for a wide, high-quality output, two takes when the budget allows and one per subject otherwise; a cameo subject gets its own sheet (field names and size caps from `model_schema_get`).
  - The prompt covers identity details, the outfit or materials, the rendering style with explicit NOTs (not Pixar, not chibi), the layout, and lighting that matches the video's sets.
  - Look at it. Crop identity references from it (face, full-body 3/4) with ffmpeg and upload them.
  - For a real person, use only their own photos with their consent. Never generate other real people.
- **Style frames: one per shot.** Before any video, generate a still at the canvas ratio (`CANVAS`: 16:9, or 9:16 on a portrait project) for every planned shot in its section's look. Same model, with the sheet and an identity crop as references, the same ratio, high quality (fields from `model_schema_get`). Every prompt needs:
  - the look's art style, named explicitly ("drawn only with glowing gold light lines on pure black", "flat tangerine cyc", "B&W 35mm film")
  - composition that leaves **negative space for the type** ("framed on the right third, left two thirds empty"; on portrait, "face at about 40% of the frame height, empty space above it for type, nothing that must read in the bottom third"). On portrait, keep the face and the type space inside the safe band: platform UI covers roughly the top 14%, the bottom 35%, 6% on the left and 13% on the right (authoring-time values, the defaults of `E.SAFE`). The subject's body may run below it.
  - a set the compositor can use (black void with a hard rim light, a flat color cyc, or a white high-key set)
  - "No text, no letters, no logos" (all real type is done in code)
  - Contact-sheet all the frames and show the user before spending on video. A style frame is the cheapest paid step; price the video steps with `dry_run`, one run per distinct payload (clip length and resolution move the price), and total them. To quote the whole video before any spend, price the video payloads with an uploaded reference photo standing in for each frame: in the live test those quotes matched every video job's billed `cuCost`. Ask the user to approve the look and the total before launching; unattended, stop if no budget or no explicit team and project was given.
- **Props and 3D.** Image-to-3D (an untextured, low-face-count mesh is enough; read the options from the schema) gives a GLB for three.js (wireframe → solid reveals, a creature that peels off a page).
- Download with `asset_download` → `curl -L`. Always look at the results before going on.

## 4. Scenario: footage routes

Find current models with `recommend`, passing each lane's value as `capability` (`img2video`, `audio2video`; a reference-image, reference-audio dance is `img2video` with the reference features; lip-sync correction is `video2video`, with the job in the prompt) and read each schema with `model_schema_get`. Pass a clip length through the `duration` argument, never in the prompt: `recommend` read "under 10 s" there as a latency limit. Footage is generated at the canvas ratio: image-to-video follows a 9:16 first frame where the schema says the frame sets the shape, other routes take the ratio through the schema's aspect field, and a member that offers neither gets a new `recommend`, never a cropped 16:9 clip. The parameters below are concepts, not payloads: the schema wins. `dry_run` every distinct payload and total the quotes. Launch in waves within the team's concurrency ceiling with `wait:false` (a top-level `model_run` argument, beside `parameters`), then `jobs_wait` with the wave's job ids, re-called with `pending_job_ids` until none are pending. A launch refused with 429 `parallel-custom-jobs` created no job: relaunch it in the next wave. A wait timeout is not a failure: never relaunch on one, and never poll with `job_get`.

| shot                              | route                                                                                                                          | key params                                                                                                                               |
| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
| **any action from a style frame** | image-to-video                                                                                                                 | first frame, duration, resolution, the canvas ratio where the schema has an aspect field, native audio off                               |
| **dance locked to the song**      | reference-image and reference-audio video + a synthesized beat track                                                           | reference images (sheet, crop or frame), the beat track as reference audio, duration, the canvas ratio, native audio off; then beat-warp |
| **lip-sync** (preferred)          | audio-to-video from a style frame and a vocal slice                                                                            | first frame, the vocal slice as audio, resolution, the canvas ratio where the schema has one, an optional camera-motion preset           |
| **lip-sync** (alternative)        | reference-audio video with the vocal stem, then a lip-sync correction model (video and audio in, cut-off sync mode if offered) | measure with `lipsync_check.py`                                                                                                          |
| **a moment code does better**     | build it in JS instead (node networks, constellations, charts, UI, particles, any text)                                        | generated versions of these looked worse and were replaced                                                                               |

- **Prompt shape** (every good shot had these parts):
  1. what each `@image`/`@audio` input is for ("Animate this exact image, keep its exact art style of …")
  2. one dominant action, with timing cues in clip seconds if needed ("near the end she slowly closes her eyes")
  3. one camera move
  4. the set and light, held constant ("background stays pure black", "rim light stays constant")
  5. exclusions ("No text, no captions, no logos, no watermark, no other people. Silent footage.")
- **Synth beat track for dances:** `.venv/bin/python tools/synth_beat.py <slot> <dur> assets/audio_slices/<name>_beat.wav`. Upload it and say "@audio1 is a percussion timing guide: every hit lands exactly on the kicks and snares". The clip then belongs at song time `slot`.
- **Reference images keep a normal shape.** A full-body crop narrower than 0.4 (width over height) failed one member's reference check after `dry_run` had priced the payload without complaint. Pad tall crops toward the canvas ratio before uploading.
- **Vocal slices for lip-sync** come from the vocal stem as WAV (mp3 adds priming delay): `vocal_stem="$(.venv/bin/python tools/audio_paths.py)" && ffmpeg -i "$vocal_stem" -ss <slot> -t <dur> -ac 1 -ar 44100 -c:a pcm_s16le assets/audio_slices/<name>.wav`. The resolver reads the approved master from `engine/data/audio.json` and fails if its stem is missing. After an audio edit, regenerate affected slices from the new stem. The clip belongs at `slot`. Lip-sync prompts ask for "precise, expressive lip sync, strong mouth shapes on every syllable", with the mouth always visible and hands out of frame.
- **Never send the song itself** to a video model. Moderation rejects it, and `generateAudio:true` can reproduce the vocal and fail the job. Use beat tracks and vocal stems only.
- **Shot mix to plan:**
  - a hook close-up for frame 0
  - 2-3 lip-sync shots in different looks
  - 2-4 full-body dances or formations
  - a long-shot action (a sprint or leap)
  - a spin or signature move
  - a few "concept" shots that illustrate specific lyrics
  - a finale
  - Include full-body long shots as well as close-ups. About 17 frames, 17 videos and 6 lip-syncs came to about 6k CU at 720p; a 1080p portrait remake with 24 frames and 27 clips came to about 15k.
- **Revisions:** a shot that doesn't work gets regenerated in a different look or replaced by code. Don't patch it with effects. For a bad cut-out (fine detail such as fingers on a keyboard, or hair on white), regenerate the shot in a style that keys cleanly (pixel art, flat color), or show the full plate inside a window.

## 5. Footage prep: frames, mattes, tracking, checks

```bash
bash tools/prep_clip.sh                       # every assets/clips/*.mp4 → assets/video/<clip>/ frames + m_*.png mattes + track.json
bash tools/prep_clip.sh lip_a dance_b         # or only some clips
.venv/bin/python tools/fixmatte.py <clip>     # flat color-cyc clips: union a chroma key with the Vision matte (keeps floor shadows out)
.venv/bin/python tools/fixmatte.py <clip> --white   # white high-key clips (not when the subject wears white: a white hoodie reads as background and the matte gets holes)
.venv/bin/python tools/beatwarp.py <clip> <slot>                                     # dance clips: meta.warp time remap onto the song's accents
.venv/bin/python tools/dance_sync_check.py assets/clips/<clip>.mp4 <slot>             # motion-onset vs beat correlation
.venv/bin/python tools/lipsync_check.py assets/clips/<clip>.mp4 assets/audio_slices/<vocal>.wav out/ls_<clip>.png
ffmpeg -i assets/clips/<clip>.mp4 -vf "fps=2,scale=320:-1,tile=6x4" -frames:v 1 out/sheet_<clip>.jpg   # then Read it
```

- **Mattes:** `tools/matte` uses Apple Vision's foreground-instance mask. It is free and takes about 0.1 s per frame, writing 8-bit `m_#####.png` (white = subject).
  - Open a few mattes and check them over a bright color, especially hair, fingers and anything near a white background.
  - Matte sources, in order. Apple Vision first: free, fast and good on people. When `tools/matte` is missing (no macOS, or an unaccepted Xcode license blocks `swiftc`) or its matte fails on a clip (white subject on a white set, props, hair), use Scenario: `recommend` with `video2video` for video background removal, ask for a solid magenta background rather than transparency (a transparent request can come back in a container without alpha), `dry_run` it, run it on the clip's asset, `asset_download` the result, then `.venv/bin/python tools/matte_keyed.py <clip> <keyed.mp4> --force`. MediaPipe last: `.venv/bin/python tools/matte_fallback.py <clip> ...` is free but soft, keeps people and loses props and fine detail.
  - Rerun `fixmatte.py` on flat-cyc clips after replacing a matte, then `track.py`.
- **Pixel-art subjects:** soft mattes look dirty on pixel edges. Threshold to a hard binary mask on the pixel grid and add a 1-cell ink outline.
- **Tracking:** `track.py` writes MediaPipe pose (33 landmarks) plus the matte bbox, centroid, area and top point per frame. Pose fails on non-human subjects and on wide group shots, but the matte-derived fields still work.
- Write each clip's content, key moments (in clip time), slot and matte quality into TREATMENT.md's footage map. Agents work from it.

## 6. Engine and agents

Run `tools/track.py` after the mattes exist: it writes `box`, `c` and `top` from the matte, and a clip tracked before them has pose only.

- **Before launching agents:**
  - Write STYLE.md and TREATMENT.md (see SKILL.md), and AGENTS_BRIEF.md from `references/agent_brief_template.md`. Copy ENGINE_API.md into the project.
  - Fill in `engine/timeline.js` (`CANVAS`, sections on downbeats, `HUD`, `POST`) and set `PAL` in `core.js`.
  - Write a tiny `_smoke.js`: plate → type behind the subject → matte → tracked label. Render one still and a 5 s clip, then run `av_sync_check.py`.
- **Launch** one `general-purpose` agent per section in a single message (background). Each prompt names its file, time range, lyrics, must-have ideas and footage, and says to read the brief first. Log the agent ids in `analysis/agents.md`, so revisions can resume them.
- **Review** each report as it arrives: build a contact sheet of its best stills and Read it. Fix engine-level problems yourself and tell every agent what changed.

## 7. Assembly, grade and verification

```bash
node tools/render.mjs --fps 60 --scale 1 --workers 3 --crf 17 --out out/final/v1.mp4 [--variant b] [--to <end >]
.venv/bin/python tools/av_sync_check.py out/final/v1.mp4 0                                          # expect audio 0.0 ms; visual 0..+1 frame
ffmpeg -i out/final/v1.mp4 -vf "freezedetect=n=0.003:d=0.25" -map 0:v -f null - 2>&1 | grep freeze_ # frozen footage or graphics
ffmpeg -i out/final/v1.mp4 -vf "fps=2,scale=384:-1" -q:v 4 out/watch/f_%04d.jpg                     # then tile 8x5 per 20 s (`tools/sheet.py <out.jpg> <cols> <img>...`) and Read every sheet
```

- **Freezes:** check every hit from `freezedetect`. A held footage frame (a clip that ran out, or a slot clip shown before its slot) was rejected as "static". Intentional holds of pure graphics are fine if something still moves.
- Check each section boundary at −1/0/+1 frames.
- **Portrait safe band:** for each 20 s window, `node tools/still.mjs --range <a>:<a+20>:1 --scale 0.5 --safe --dir out/safe/<a> --sheet safe --cols 5`, then Read each sheet. Every word, logo and face sits inside the dashed band; full-bleed footage and graphics may run behind it.
- Do a global look pass: a still sheet across all sections, then tune `POST`, the palette and the HUD theme before the final render.

## 8. Delivery

```bash
# landscape canvas: the X-ready file (YouTube takes the master itself)
ffmpeg -i out/final/vN.mp4 -c:v libx264 -preset slow -b:v 21M -maxrate 25M -bufsize 42M -pass 1 -an -f mp4 /dev/null
ffmpeg -i out/final/vN.mp4 -c:v libx264 -preset slow -b:v 21M -maxrate 25M -bufsize 42M -pass 2 -pix_fmt yuv420p -profile:v high -c:a aac -b:a 192k -ar 44100 -movflags +faststart out/final/vN_x_1080p60.mp4
# portrait canvas: one vertical upload for TikTok, Reels and Shorts (the platforms re-encode, so keep quality high)
ffmpeg -i out/final/vN.mp4 -c:v libx264 -preset slow -crf 17 -pix_fmt yuv420p -profile:v high -c:a aac -b:a 192k -ar 44100 -movflags +faststart out/final/vN_vertical_1080x1920.mp4
# teaser: END is the teaser length in seconds, FADE = END - 0.6
ffmpeg -i out/final/vN.mp4 -t "$END" -af "afade=t=out:st=$FADE:d=0.6" out/final/vN_teaser.mp4
```

- **Vertical length:** platform caps on vertical uploads change often (Shorts, Reels and TikTok each set their own); check the target's current limit before the final render, and cut the teaser when the song runs longer.
- **Two ratios:** the renders are `vN_16x9.mp4` and `vN_9x16.mp4` (§9). Run each encode on its own canvas's render and carry the suffix into every output (`vN_9x16_vertical_1080x1920.mp4`, `vN_9x16_teaser.mp4`), so nothing collides.
- **Name outputs distinctly.** macOS filenames are case-insensitive, so an encode named like its input with different case overwrites the input mid-encode.
- **Variants:** render each with `--variant`, and name the files by what differs (`_ending_b.mp4`).
- **Stills:** `node tools/still.mjs --t <times> --scale 1 --dir out/stills` gives lossless PNGs at the canvas size (1920×1080 or 1080×1920) straight from the engine.
- **Style sheet / making-of:** publish an artifact or doc with the palette, type, subject sheets, source-vs-final footage pairs, a frame index, the pipeline and the iteration count.

## 9. Two ratios from one project

Only when the user asks for both: one canvas is the default. Everything in code is shared, so the analysis, lyrics, STYLE, TREATMENT, scenes and HUD render on the second canvas at no generation cost. Footage is not.

- **Pick the primary canvas,** the platform that matters most, and set it as `CANVAS`. When the user names both without ranking them, ask; unattended, landscape. Style frames and footage are generated at its ratio, as in §3 and §4.
- **Per shot on the secondary canvas,** cheapest first:
  1. **Window it.** Play the primary clip whole inside the other frame (a band, a card, a split panel or a type window, per `motion_library.md` §5) and let graphics fill the rest. Free, and it reads as design.
  2. **Regenerate it** at the secondary ratio: a new style frame composed for that ratio (same references, same look), then the same footage route. It costs what the original cost, so `dry_run` it. Keep it for the shots that must fill the frame: the frame-0 hook, lip-sync close-ups and the finale.
- **Never cover-crop 16:9 to fill 9:16.** It keeps under a third of the width (608 of 1920 px on a 1080p clip, 405 of 1280 on 720p), upscaled about 1.8 to 2.7 times, and it usually cuts the subject.
- **Budget:** quote both canvases before launching: the primary footage plus the regenerated shots. Tell the user which shots are windowed on which canvas.
- **Scenes:** every plate takes `fit: E.plateFit(E, v)` (and the same options in `plateToScreen`), which fills the frame when the clip matches the canvas and windows it when it does not; `drawPlate`'s own default is `cover`, which would crop. A regenerated shot is a new clip with its own prep (§5), picked by canvas in the same scene (`E.loadVideo(E.ORIENT === 'portrait' ? 'hook_v' : 'hook')`). Give a section a second scene file only when its layout diverges (a wide poster slam), and tag the pair in `TIMELINE`: `canvas: 'landscape'` on one, `canvas: 'portrait'` on the other.
- **Render and verify each canvas:** `node tools/render.mjs --fps 60 --scale 1 --workers 3 --crf 17 --out out/final/vN_16x9.mp4`, then the same with `--canvas portrait --out out/final/vN_9x16.mp4` (swap them when portrait is the primary). Run §7's checks on each file, plus the safe-band sheets on the portrait one (`still.mjs --canvas portrait --safe` when the primary is landscape).
- **Deliver** one set per canvas, suffixed by ratio as in §8.

"""Song -> engine/data/audio.json: beats, downbeats, kick/snare/vocal onsets, 60 fps envelopes. Also separates the vocal stem.
usage: python scripts/audio_analysis.py audio/master.mp3
Writes: stems/<name>/vocals.wav + no_vocals.wav (Demucs), engine/data/audio.json (with "master" = the input path).
The master itself is never modified."""
import sys, os, json, subprocess, numpy as np, librosa

if len(sys.argv) < 2: sys.exit(__doc__)
master = sys.argv[1]; name = os.path.splitext(os.path.basename(master))[0]
stem_dir = f'stems/htdemucs/{name}'
if not os.path.exists(f'{stem_dir}/vocals.wav'):
    subprocess.run([sys.executable, '-m', 'demucs', '--two-stems=vocals', '-n', 'htdemucs', '-o', 'stems', master], check=True)
FPS_ENV = 60
from beat_this.inference import File2Beats


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


beats, downbeats = File2Beats(checkpoint_path='final0', device='cpu', dbn=False)(master)
beats = [float(b) for b in beats]; downbeats = [float(b) for b in downbeats]
bpm = float(60 / np.median(np.diff(beats)))
voc, sr = librosa.load(f'{stem_dir}/vocals.wav', sr=22050, mono=True)
inst, _ = librosa.load(f'{stem_dir}/no_vocals.wav', sr=22050, mono=True)
mix, _ = librosa.load(master, sr=22050, mono=True)
hop = sr // FPS_ENV
def env(y):
    r = librosa.feature.rms(y=y, frame_length=hop * 2, hop_length=hop)[0]; return np.clip(r / (np.percentile(r, 99) + 1e-9), 0, 1.5)
S = np.abs(librosa.stft(inst, n_fft=2048, hop_length=hop)); fr = librosa.fft_frequencies(sr=sr, n_fft=2048)
def band(lo, hi):
    b = S[(fr >= lo) & (fr < hi)].sum(0); return np.clip(b / (np.percentile(b, 99) + 1e-9), 0, 1.5)
def band_onsets(lo, hi, k=1.2):
    Sx = np.abs(librosa.stft(inst, n_fft=2048, hop_length=256)); f2 = librosa.fft_frequencies(sr=sr, n_fft=2048)
    sub = librosa.amplitude_to_db(Sx[(f2 >= lo) & (f2 < hi)])
    o = np.concatenate([[0], np.maximum(0, np.diff(sub, axis=1)).mean(0)]).astype(np.float64)
    pk = librosa.util.peak_pick(o, pre_max=3, post_max=3, pre_avg=10, post_avg=10, delta=float(k * o.std()), wait=8)
    return [float(p * 256 / sr) for p in pk]
out = {
    'master': master, 'duration': float(len(mix) / sr), 'bpm': bpm, 'beats': beats, 'downbeats': downbeats,
    'kicks': band_onsets(30, 130), 'snares': band_onsets(1500, 5000),
    'vocal_onsets': [float(x) for x in librosa.onset.onset_detect(y=voc, sr=sr, hop_length=256, units='time', delta=0.08)],
    'env_fps': FPS_ENV,
    'env': {k: np.round(v, 3).tolist() for k, v in {'mix': env(mix), 'vocal': env(voc), 'kick': band(30, 130), 'snare': band(1500, 5000), 'hats': band(7000, 11000), 'bass': band(40, 250)}.items()},
}
os.makedirs('engine/data', exist_ok=True); save_json(out, 'engine/data/audio.json')
print(f"bpm {bpm:.1f}, {len(beats)} beats, {len(downbeats)} downbeats, {len(out['kicks'])} kicks, {len(out['snares'])} snares, dur {out['duration']:.2f}s")
print('downbeats:', ' '.join(f'{x:.2f}' for x in downbeats))

"""Resolve the vocal stem belonging to the approved master, never an older take."""
import json
from pathlib import Path


def vocal_stem():
    with open('engine/data/audio.json') as source:
        master = json.load(source)['master']
    path = Path('stems/htdemucs') / Path(master).stem / 'vocals.wav'
    if not path.is_file():
        raise FileNotFoundError(f'{path}: run audio_analysis.py on the approved master first')
    return str(path)


if __name__ == '__main__':
    print(vocal_stem())

"""Verify A/V sync of a rendered mp4.
1) audio offset: cross-correlate the mp4's decoded audio with the master (expects 0 ms)
2) visual accents vs beats: visual change energy (frame diff) vs kick/snare/downbeat train from audio.json; best lag (expects ~0..+1 frame)
usage: python tools/av_sync_check.py out.mp4 [start_sec_of_mp4_in_song=0]"""
import sys, json, numpy as np, librosa, cv2


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


if len(sys.argv) < 2: sys.exit(__doc__)
mp4 = sys.argv[1]; t0 = float(sys.argv[2]) if len(sys.argv) > 2 else 0.0
sr = 8000
a, _ = librosa.load(mp4, sr=sr, mono=True)
MASTER = load_json('engine/data/audio.json')['master']
m, _ = librosa.load(MASTER, sr=sr, mono=True, offset=t0, duration=len(a)/sr)
n = min(len(a), len(m), sr*30); A = a[:n] - a[:n].mean(); M = m[:n] - m[:n].mean()
lags = range(-400, 401, 4); best = max(lags, key=lambda L: np.dot(A[1000+max(0,L):n-1000+min(0,L)], M[1000-min(0,L):n-1000-max(0,L)]))
# refine
fine = range(best-4, best+5); best = max(fine, key=lambda L: np.dot(A[1000+max(0,L):n-1000+min(0,L)], M[1000-min(0,L):n-1000-max(0,L)]))
print(f"audio offset vs master: {best/sr*1000:+.1f} ms (+ = mp4 audio late)")
cap = cv2.VideoCapture(mp4); fps = cap.get(cv2.CAP_PROP_FPS); prev = None; D = []
while True:
    ok, f = cap.read()
    if not ok: break
    g = cv2.cvtColor(cv2.resize(f, (192, 108)), cv2.COLOR_BGR2GRAY).astype(np.float32)
    D.append(0 if prev is None else np.abs(g - prev).mean() + abs(g.mean() - prev.mean()) * 3); prev = g
D = np.array(D); J = load_json('engine/data/audio.json')
ev = [x - t0 for x in J['kicks'] + J['snares'] + J['downbeats'] if t0 <= x < t0 + len(D)/fps]
tr = np.zeros(len(D))
for e in ev:
    i = int(round(e*fps))
    if 0 <= i < len(D): tr[i] = 1
z = lambda x: (x - x.mean())/(x.std() + 1e-9); vz = z(np.maximum(0, np.diff(D, prepend=D[0]))); tz = z(tr)
L = range(-6, 7); cs = [np.corrcoef(np.roll(vz, -l), tz)[0, 1] for l in L]
b = list(L)[int(np.argmax(cs))]
print(f"visual accents vs audio events: best lag {b:+d} frames ({b/fps*1000:+.0f} ms, + = picture late), corr@0={cs[6]:.3f} corr@best={max(cs):.3f}")
print("per-lag:", " ".join(f"{l:+d}:{c:.2f}" for l, c in zip(L, cs)))

"""Beat-warp a dance clip: time-remap so its motion accents land on the song's kicks/snares.
usage: python tools/beatwarp.py clipname slot_start   (clip already prepped in assets/video/<clip>)
Adds meta.warp = [[song_local_t, clip_t], ...] (monotonic, slope clamped 0.75..1.33)."""
import sys, json, numpy as np


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


from PIL import Image
if len(sys.argv) < 3: sys.exit(__doc__)
clip, slot = sys.argv[1], float(sys.argv[2]); d = f'assets/video/{clip}'; M = load_json(f'{d}/meta.json')
fps, n = M['fps'], M['n']; prev = None; D = []
for i in range(1, n+1):
    g = np.asarray(Image.open(f'{d}/f_{i:05d}.jpg').convert('L').resize((160, 90)), dtype=np.float32)
    D.append(0 if prev is None else np.abs(g-prev).mean()); prev = g
D = np.array(D); acc = np.maximum(0, np.diff(D, prepend=D[0])); acc = np.convolve(acc, [0.25, 0.5, 0.25], 'same')
from scipy.signal import find_peaks

pk, _ = find_peaks(acc, distance=int(0.25*fps), prominence=np.percentile(acc, 75))
peaks = pk / fps
A = load_json('engine/data/audio.json')
ev = np.array(sorted(set([round(x-slot, 3) for x in A['kicks'] + A['snares'] + A['beats'] if slot <= x < slot + n/fps])))
pairs = []
for p in peaks:
    j = np.argmin(np.abs(ev - p))
    if abs(ev[j]-p) < 0.16: pairs.append((ev[j], p))
pairs = sorted(set(pairs)); anchors = [(0.0, 0.0)]
for s, c in pairs:
    ls, lc = anchors[-1]
    if s - ls < 0.2: continue
    slope = (c-lc)/(s-ls)
    if 0.75 <= slope <= 1.33: anchors.append((s, c))
end = n/fps; ls, lc = anchors[-1]; anchors.append((ls + (end-lc), end))
M['warp'] = [[round(a, 4), round(b, 4)] for a, b in anchors]; save_json(M, f'{d}/meta.json', indent=1)
shift = [round((c - s)*1000) for s, c in anchors[1:-1]]
print(f'{clip}: {len(peaks)} motion peaks, {len(pairs)} matched, {len(anchors)-2} anchors, shifts ms: {shift}')

"""Motion-onset vs beat correlation for dance clips. usage: python tools/dance_sync_check.py clip.mp4 slot_start"""
import sys, json, numpy as np, cv2


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


if len(sys.argv) < 3: sys.exit(__doc__)
vid, slot = sys.argv[1], float(sys.argv[2])
cap = cv2.VideoCapture(vid); fps = cap.get(cv2.CAP_PROP_FPS); prev=None; D=[]
while True:
    ok, f = cap.read()
    if not ok: break
    g = cv2.cvtColor(cv2.resize(f,(160,90)), cv2.COLOR_BGR2GRAY).astype(np.float32)
    D.append(0 if prev is None else np.abs(g-prev).mean()); prev=g
D=np.array(D); t=np.arange(len(D))/fps
A=load_json('engine/data/audio.json')
# accent train: kicks+snares near beats within slot
ev=[x-slot for x in A['kicks']+A['snares'] if slot<=x<slot+len(D)/fps]
train=np.zeros(len(D));
for e in ev:
    i=int(round(e*fps));
    if 0<=i<len(D): train[i]=1
k=np.exp(-np.arange(0,6)/2.0); train=np.convolve(train,k)[:len(D)]
# motion "acceleration" onsets = positive derivative of motion energy
m=np.maximum(0,np.diff(D,prepend=D[0]))
z=lambda x:(x-x.mean())/(x.std()+1e-9)
lags=range(-8,9); cs=[np.corrcoef(np.roll(z(m),-L),z(train))[0,1] for L in lags]
b=list(lags)[int(np.argmax(cs))]
# null: shuffled/shifted baseline
null=[np.corrcoef(np.roll(z(m),s),z(train))[0,1] for s in range(15,len(D)-15,7)]
print(f"fps={fps} corr@0={cs[8]:.3f} best_lag={b/fps*1000:+.0f}ms corr@best={max(cs):.3f} null_mean={np.mean(null):.3f} null_std={np.std(null):.3f}")

"""Union a hue-based chroma key with the Vision matte for flat colour-cyc clips (keeps floor shadows as background).
usage: python tools/fixmatte.py <clip> ; backs up originals to m_vision/"""
import sys, os, glob, numpy as np, cv2
if len(sys.argv) < 2: sys.exit(__doc__)
clip = sys.argv[1]; WHITE = '--white' in sys.argv; d = f'assets/video/{clip}'; os.makedirs(f'{d}/m_vision', exist_ok=True)
fs = sorted(glob.glob(f'{d}/f_*.jpg'))
im0 = cv2.imread(fs[0]).astype(np.float32) / 255
h, w = im0.shape[:2]
border = np.concatenate([im0[:h//8, :].reshape(-1, 3), im0[:, :w//12].reshape(-1, 3), im0[:, -w//12:].reshape(-1, 3)])
bg = np.median(border, 0); bgc = bg - bg.mean(); bgc /= np.linalg.norm(bgc) + 1e-6
for f in fs:
    im = cv2.imread(f).astype(np.float32) / 255
    c = im - im.mean(2, keepdims=True); n = np.linalg.norm(c, axis=2) + 1e-6
    cos = (c @ bgc) / n; sat = im.max(2) - im.min(2)
    bgness = np.clip((cos - 0.80) / 0.12, 0, 1) * np.clip((sat - 0.10) / 0.12, 0, 1)
    if WHITE: bgness = np.clip((im.min(2) - 0.80) / 0.10, 0, 1) * np.clip((0.12 - sat) / 0.08, 0, 1)
    fg = (1 - bgness)
    fg = cv2.GaussianBlur(fg, (0, 0), 1.2)
    mp = f.replace('/f_', '/m_').replace('.jpg', '.png'); bk = f'{d}/m_vision/' + os.path.basename(mp)
    if not os.path.exists(bk): os.rename(mp, bk)
    v = cv2.imread(bk, cv2.IMREAD_GRAYSCALE).astype(np.float32) / 255
    if v.shape != fg.shape: v = cv2.resize(v, (w, h))
    m = np.maximum(v, fg) if not WHITE else np.where(fg > 0.5, np.maximum(v, fg), np.minimum(v, fg))
    cv2.imwrite(mp, (m * 255).astype(np.uint8))
print(clip, 'bg', np.round(bg * 255).astype(int), len(fs))

"""Verify lip sync of a clip against its reference vocal slice.
Measures mouth openness per frame (MediaPipe FaceMesh inner-lip distance / face height), correlates with the vocal RMS envelope,
searches the best lag in +-400ms. Prints corr at lag 0, best lag, and writes a plot.
usage: python tools/lipsync_check.py clip.mp4 vocal_slice.wav out.png"""
import os, sys, numpy as np, cv2, librosa, mediapipe as mp, matplotlib; matplotlib.use('Agg'); import matplotlib.pyplot as plt
if len(sys.argv) < 4: sys.exit(__doc__)
vid, wav, png = sys.argv[1:4]
if os.path.abspath(png) in (os.path.abspath(vid), os.path.abspath(wav)): sys.exit('out.png must not be one of the inputs')
cap = cv2.VideoCapture(vid); fps = cap.get(cv2.CAP_PROP_FPS); fm = mp.solutions.face_mesh.FaceMesh(static_image_mode=False, refine_landmarks=True, max_num_faces=1)
M = []
while True:
    ok, fr = cap.read()
    if not ok: break
    r = fm.process(cv2.cvtColor(fr, cv2.COLOR_BGR2RGB))
    if r.multi_face_landmarks:
        L = r.multi_face_landmarks[0].landmark; op = abs(L[13].y - L[14].y); fh = abs(L[10].y - L[152].y) + 1e-6; M.append(op / fh)
    else: M.append(np.nan)
M = np.array(M); t = np.arange(len(M)) / fps
y, sr = librosa.load(wav, sr=16000); hop = int(sr / fps)
env = librosa.feature.rms(y=y, frame_length=hop*2, hop_length=hop)[0][:len(M)]
env = np.pad(env, (0, max(0, len(M)-len(env))))
m = np.nan_to_num(M, nan=np.nanmean(M)); z = lambda x: (x - x.mean()) / (x.std() + 1e-9)
mz, ez = z(m), z(env)
lags = range(-int(0.4*fps), int(0.4*fps)+1); cs = [np.corrcoef(np.roll(mz, -L), ez)[0,1] for L in lags]
best = list(lags)[int(np.argmax(cs))]
print(f"frames={len(M)} fps={fps:.2f} face_found={np.mean(~np.isnan(M)):.2f} corr@0={cs[len(cs)//2]:.3f} best_lag={best/fps*1000:+.0f}ms corr@best={max(cs):.3f}")
fig, ax = plt.subplots(2,1, figsize=(14,5)); ax[0].plot(t, mz, label='mouth'); ax[0].plot(t, ez, label='vocal rms', alpha=.7); ax[0].legend(); ax[1].plot([l/fps*1000 for l in lags], cs); ax[1].set_xlabel('lag ms (+ = mouth late)'); plt.tight_layout(); plt.savefig(png, dpi=70)

"""Word-level lyric timing -> engine/data/lyrics.json
usage:
  python scripts/lyrics_align.py transcribe [--prompt "vocabulary hints, names, jargon"]  # writes analysis/lyrics_raw.json + prints lines
  python scripts/lyrics_align.py align lyrics.txt [--chant outro:GO]                      # maps corrected lyrics onto the timings
Each transcribe pass also keeps its own copy (lyrics_raw_plain.json without --prompt, lyrics_raw_hints.json with it), so the
plain and hinted passes can be compared; align reads lyrics_raw.json, the latest pass.
lyrics.txt format: '# section: name' lines, then one sung line per row (you correct the transcript by hand; supplied lyrics always win).
--chant: that section's lines are counted, not aligned. Each vocal peak between the lines around the section becomes one chant
word, up to as many words as the section's lines hold (all peaks to the next line when it has none).
Method (what held up on sung vocals): faster-whisper large-v3 word timestamps on the Demucs vocal stem, sequence-matched onto the
corrected lyrics, starts snapped to vocal onsets within 80 ms, then a fix for short first words pulled early by a held note.
wav2vec/MMS forced alignment ran ~200 ms late on singing, so it is not used. Always eyeball the result with plot_lyrics.py."""
import os, json, re, difflib, argparse, numpy as np


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


def stem():
    from audio_paths import vocal_stem
    return vocal_stem()

def raw_paths(prompt):
    return ['analysis/lyrics_raw.json', f"analysis/lyrics_raw_{'hints' if prompt else 'plain'}.json"]

def transcribe(prompt):
    from faster_whisper import WhisperModel
    m = WhisperModel('large-v3', device='cpu', compute_type='int8')
    segs, _ = m.transcribe(stem(), language='en', word_timestamps=True, beam_size=5, condition_on_previous_text=False, initial_prompt=prompt or None, temperature=0)
    out = []
    for s in segs:
        out.append({'start': s.start, 'end': s.end, 'text': s.text.strip(), 'words': [{'w': w.word.strip(), 's': round(w.start, 3), 'e': round(w.end, 3)} for w in s.words]})
        print(f'[{s.start:7.2f}-{s.end:7.2f}] {s.text.strip()}', flush=True)
    os.makedirs('analysis', exist_ok=True)
    for p in raw_paths(prompt): save_json(out, p, indent=1)

def align(path, chant=None, no_fix=()):
    A = load_json('engine/data/audio.json'); vo = np.array(A['vocal_onsets'])
    W = [w for s in load_json('analysis/lyrics_raw.json') for w in s['words']]
    norm = lambda s: re.sub(r'[^a-z0-9]', '', s.lower())
    lines, sec, sec_at = [], None, {}
    with open(path) as fh:
        raw = fh.readlines()
    for L in raw:
        L = L.strip()
        if not L: continue
        if L.startswith('# section:'): sec = L.split(':', 1)[1].strip(); sec_at.setdefault(sec, len(lines)); continue
        lines.append({'section': sec, 'text': L, 'words': L.split()})
    chant_sec, chant_word = (chant.split(':') + ['GO'])[:2] if chant else (None, None)
    C = [(li, wi, norm(w)) for li, l in enumerate(lines) if l['section'] != chant_sec for wi, w in enumerate(l['words'])]
    sm = difflib.SequenceMatcher(a=[c[2] for c in C], b=[norm(w['w']) for w in W], autojunk=False)
    T = [None] * len(C)
    for tag, i1, i2, j1, j2 in sm.get_opcodes():
        if tag == 'equal':
            for k in range(i2 - i1): T[i1 + k] = (W[j1 + k]['s'], W[j1 + k]['e'])
        elif tag == 'replace':
            # spread over at most as many transcript words as lyric words: when the supplied lyrics cover only part of
            # the song, the tail of the transcript is not theirs (it once stretched a last word to the song's end)
            n = i2 - i1; s, e = W[j1]['s'], W[min(j2, j1 + n) - 1]['e']
            for k in range(n): T[i1 + k] = (s + (e - s) * k / n, s + (e - s) * (k + 1) / n)
    for i in range(len(C)):
        if T[i] is None:
            p = next((T[j] for j in range(i - 1, -1, -1) if T[j]), (0, 0)); T[i] = (p[1], p[1] + 0.25)
    for i, (li, wi, _) in enumerate(C):
        s, e = T[i]
        if len(vo):
            j = np.argmin(abs(vo - s))
            if abs(vo[j] - s) < 0.08: s = float(vo[j])
        lines[li].setdefault('_w', []).append({'w': lines[li]['words'][wi], 's': round(s, 3), 'e': round(max(e, s + 0.06), 3)})
    out = []
    for l in lines:
        if l['section'] == chant_sec: continue
        ws = l.pop('_w')
        for i in range(len(ws) - 1):  # short first word pulled early by the previous held note
            if ws[i + 1]['s'] - ws[i]['s'] > 0.9 and len(ws[i]['w'].strip(',.?!')) <= 4 and ws[i]['w'].strip(',.?!') not in no_fix:
                c = vo[(vo > ws[i + 1]['s'] - 0.55) & (vo < ws[i + 1]['s'] - 0.1)]
                ws[i]['s'] = round(float(c[0]) if len(c) else ws[i + 1]['s'] - 0.28, 3)
        for i, w in enumerate(ws):
            if i + 1 < len(ws): w['e'] = round(min(max(w['e'], w['s'] + 0.12), ws[i + 1]['s']), 3)
            w['e'] = round(min(w['e'], w['s'] + 2.0), 3)  # no sung word holds past 2 s
        l['words'] = ws; l['s'] = ws[0]['s']; l['e'] = ws[-1]['e']; out.append(l)
    if chant_sec:  # repeated chant: one word per vocal-envelope peak between the lines around the chant section
        from scipy.signal import find_peaks
        env = np.array(A['env']['vocal']); fps = A['env_fps']; t = np.arange(len(env)) / fps
        at = sec_at.get(chant_sec, len(lines))  # a section missing from lyrics.txt chants after the last line
        before = sum(1 for li, l in enumerate(lines) if li < at and l['section'] != chant_sec)
        t0 = out[before - 1]['e'] if before else 0
        t1 = out[before]['s'] if before < len(out) else np.inf
        m = (t > t0) & (t < t1)  # contiguous: dropping silent samples would pull peaks together and the distance rule then drops words
        pk, _ = find_peaks(env[m], height=0.3, distance=int(0.17 * fps), prominence=0.15)
        gos = [round(float(t[m][p]) - 0.06, 3) for p in pk]
        n = sum(len(l['words']) for l in lines if l['section'] == chant_sec)
        if n: gos = gos[:n]
        if gos: out.insert(before, {'section': chant_sec, 'text': (chant_word + ' ') * len(gos), 'words': [{'w': chant_word, 's': g, 'e': round(g + 0.18, 3)} for g in gos], 's': gos[0], 'e': gos[-1] + 0.3})
    os.makedirs('engine/data', exist_ok=True); save_json({'lines': out}, 'engine/data/lyrics.json', indent=1)
    for l in out: print(f"{l['s']:7.2f}-{l['e']:7.2f} {l['section'] or '':8s} {l['text'][:70]}")

if __name__ == '__main__':
    ap = argparse.ArgumentParser(); ap.add_argument('cmd'); ap.add_argument('lyrics', nargs='?'); ap.add_argument('--prompt', default='')
    ap.add_argument('--chant'); ap.add_argument('--no-fix', default='', help='comma list of short words that are legitimately long (acronyms)')
    a = ap.parse_args()
    if a.cmd == 'transcribe': transcribe(a.prompt)
    else: align(a.lyrics, a.chant, tuple(x for x in a.no_fix.split(',') if x))

"""Fallback subject matte without Apple Vision: MediaPipe selfie segmentation -> assets/video/<clip>/m_#####.png (white = subject).
usage: python tools/matte_fallback.py <clip> [<clip> ...]   (sets meta.mask=true)"""
import sys, glob, json, numpy as np, cv2
import mediapipe as mp
if len(sys.argv) < 2: sys.exit(__doc__)
seg = mp.solutions.selfie_segmentation.SelfieSegmentation(model_selection=0)
for clip in sys.argv[1:]:
    d = f'assets/video/{clip}'
    fs = sorted(glob.glob(f'{d}/f_*.jpg'))
    prev = None
    for f in fs:
        im = cv2.imread(f); rgb = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
        m = seg.process(rgb).segmentation_mask.astype(np.float32)
        m = np.clip((m - 0.35) / 0.4, 0, 1)
        m = cv2.GaussianBlur(m, (0, 0), 1.5)
        if prev is not None: m = 0.75 * m + 0.25 * prev
        prev = m
        cv2.imwrite(f.replace('/f_', '/m_').replace('.jpg', '.png'), (m * 255).astype(np.uint8))
    p = f'{d}/meta.json'
    with open(p) as fh: meta = json.load(fh)
    meta['mask'] = True
    with open(p, 'w') as fh: json.dump(meta, fh, indent=1)
    print(clip, len(fs))

"""Subject matte from a keyed video: a background-removal run that returned the subject on a solid color.
usage: python tools/matte_keyed.py <clip> <keyed.mp4|mkv|mov> [--key magenta|green|blue] [--force]
Writes assets/video/<clip>/m_#####.png (white = subject) on the clip's own frame grid and sets meta.mask.
Keying a solid color works whether or not the container kept an alpha channel, which a transparent request
does not guarantee. Refuses to replace existing mattes unless --force is given."""
import sys, glob, json, subprocess, tempfile, numpy as np, cv2

KEYS = {'magenta': (255, 0, 255), 'green': (0, 255, 0), 'blue': (0, 0, 255)}  # RGB


def key_matte(rgb, key):
    """1 = subject, 0 = key color. Distance in RGB to the key, with a soft ramp for compressed edges."""
    d = np.linalg.norm(rgb.astype(np.float32) / 255 - np.array(key, np.float32) / 255, axis=2) / np.sqrt(3)
    return np.clip((d - 0.12) / 0.18, 0, 1)


def main(argv):
    force = '--force' in argv; argv = [a for a in argv if a != '--force']
    key = 'magenta'
    if '--key' in argv:
        i = argv.index('--key'); key = argv[i + 1]; del argv[i:i + 2]
    if len(argv) < 2 or key not in KEYS: sys.exit(__doc__)
    clip, keyed = argv[0], argv[1]
    d = f'assets/video/{clip}'
    with open(f'{d}/meta.json') as fh: meta = json.load(fh)
    if glob.glob(f'{d}/m_*.png') and not force: sys.exit(f'{d} already has mattes; pass --force to replace them')
    with tempfile.TemporaryDirectory() as tmp:
        subprocess.run(['ffmpeg', '-loglevel', 'error', '-y', '-i', keyed, '-vf',
                        f"fps={meta['fps']},scale={meta['w']}:{meta['h']}:flags=lanczos", f'{tmp}/k_%05d.png'], check=True)
        ks = sorted(glob.glob(f'{tmp}/k_*.png'))
        if not ks: sys.exit(f'no frames decoded from {keyed}')
        for i in range(1, meta['n'] + 1):  # the keyed render can be a frame short or long: clamp to its last frame
            k = cv2.cvtColor(cv2.imread(ks[min(i, len(ks)) - 1]), cv2.COLOR_BGR2RGB)
            m = cv2.GaussianBlur(key_matte(k, KEYS[key]), (0, 0), 0.8)
            cv2.imwrite(f'{d}/m_{i:05d}.png', (m * 255).astype(np.uint8))
    meta['mask'] = True
    with open(f'{d}/meta.json', 'w') as fh: json.dump(meta, fh, indent=1)
    print(clip, 'mattes', meta['n'], 'from', len(ks), 'keyed frames', f'key={key}')


if __name__ == '__main__':
    main(sys.argv[1:])

"""Visual lyric-timing check: vocal-stem spectrogram with word-start markers, vocal onsets and beats.
usage: python scripts/plot_lyrics.py 1:12.5 21.5:31.5 ...   -> analysis/plots/lyrics_<a>.png  (then look at the PNGs)
Word starts should sit on the rise of a harmonic stack / consonant burst. Held notes show as long flat stacks."""
import sys, os, json, numpy as np, librosa, matplotlib; matplotlib.use('Agg'); import matplotlib.pyplot as plt
from audio_paths import vocal_stem


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


L = load_json('engine/data/lyrics.json'); A = load_json('engine/data/audio.json')
y, sr = librosa.load(vocal_stem(), sr=22050); os.makedirs('analysis/plots', exist_ok=True)
if len(sys.argv) < 2: sys.exit(__doc__)
for rng in sys.argv[1:]:
    a, b = map(float, rng.split(':')); seg = y[int(a * sr):int(b * sr)]
    S = librosa.amplitude_to_db(np.abs(librosa.stft(seg, n_fft=1024, hop_length=128)), ref=np.max)
    fig, ax = plt.subplots(2, 1, figsize=(22, 7), sharex=True, gridspec_kw={'height_ratios': [3, 1]})
    ax[0].imshow(S[:200], origin='lower', aspect='auto', extent=[a, b, 0, 200 * sr / 1024], cmap='magma', vmin=-60)
    env = np.array(A['env']['vocal']); t = np.arange(len(env)) / A['env_fps']; m = (t >= a) & (t <= b); ax[1].plot(t[m], env[m], 'k')
    for l in L['lines']:
        for w in l['words']:
            if a <= w['s'] <= b: ax[0].axvline(w['s'], color='cyan', lw=1); ax[0].text(w['s'], 3800, w['w'], color='cyan', fontsize=9, rotation=90)
    for o in A['vocal_onsets']:
        if a <= o <= b: ax[1].axvline(o, color='green', lw=1)
    for bt in A['beats']:
        if a <= bt <= b: ax[1].axvline(bt, color='gray', lw=0.6, ls=':')
    plt.tight_layout(); f = f'analysis/plots/lyrics_{a:g}.png'; plt.savefig(f, dpi=70); plt.close(); print(f)

#!/usr/bin/env bash
# Prep generated clips for the engine: frames (24 fps, native res) -> subject matte -> tracking.
# usage: bash tools/prep_clip.sh                 # every assets/clips/*.mp4
#        bash tools/prep_clip.sh lip_a dance_b    # only these clip names
# Matte: Apple Vision foreground mask via tools/matte (macOS, compiled by scaffold.sh). Elsewhere, generate mattes another way
# (e.g. Scenario video background removal) and save them as assets/video/<clip>/m_00001.png ... (white = subject).
# Afterwards, for flat colour-backdrop clips: .venv/bin/python tools/fixmatte.py <clip> [--white]
cd "$(dirname "$0")/.."
names=("$@")
if [ ${#names[@]} -eq 0 ]; then for f in assets/clips/*.mp4; do names+=("$(basename "$f" .mp4)"); done; fi
for n in "${names[@]}"; do
  f="assets/clips/$n.mp4"
  [ -f "$f" ] || {
    echo "missing $f"
    continue
  }
  w=$(ffprobe -v error -select_streams v -show_entries stream=width -of csv=p=0 "$f")
  .venv/bin/python tools/prep_video.py "$f" "$n" --fps 24 --width "$w" 2>&1 | grep -v objc | tail -1
  if [ -x tools/matte ]; then
    ./tools/matte "assets/video/$n"
    .venv/bin/python - "$n" <<'PY' 2>&1 | grep -v objc
import json,sys; p=f'assets/video/{sys.argv[1]}/meta.json'; m=json.load(open(p)); m['mask']=True; json.dump(m,open(p,'w'),indent=1)
PY
  else echo "$n: no tools/matte binary; skipping matte. Fallbacks: Scenario background removal + tools/matte_keyed.py, then tools/matte_fallback.py (references/pipeline.md §5)"; fi
  .venv/bin/python tools/track.py "$n" 2>&1 | grep -v -E "objc|INFO|WARNING|W0000|I0000" | tail -1
done
echo PREP_DONE

"""Generated clip -> engine frame sequence + analysis.
usage: python tools/prep_video.py in.mp4 clipname [--fps 30] [--width 1280] [--pose]
writes assets/video/<clip>/f_00001.jpg..., meta.json {fps,n,w,h,dur,ext}, and optional pose.json (33 landmarks/frame, normalized)."""
import os, json, shutil, subprocess, argparse
ap = argparse.ArgumentParser(); ap.add_argument('src'); ap.add_argument('clip'); ap.add_argument('--fps', type=float, default=30); ap.add_argument('--width', type=int, default=1280); ap.add_argument('--pose', action='store_true')
a = ap.parse_args()
out = f'assets/video/{a.clip}'; os.makedirs(out, exist_ok=True)
# Re-prepping replaces the take: drop its frames, mattes and matte backups so nothing stale is reused.
for f in os.listdir(out):
    if f.startswith(('f_', 'm_')) and os.path.isfile(os.path.join(out, f)): os.remove(os.path.join(out, f))
shutil.rmtree(os.path.join(out, 'm_vision'), ignore_errors=True)
subprocess.run(['ffmpeg','-loglevel','error','-y','-i',a.src,'-vf',f'fps={a.fps},scale={a.width}:-2:flags=lanczos','-q:v','2',f'{out}/f_%05d.jpg'], check=True)
n = len([f for f in os.listdir(out) if f.startswith('f_')])
pr = json.loads(subprocess.run(['ffprobe','-v','error','-show_entries','stream=width,height,duration,r_frame_rate','-of','json',f'{a.src}'],capture_output=True,text=True).stdout)['streams'][0]
from PIL import Image
w,h = Image.open(f'{out}/f_00001.jpg').size
import numpy as _np


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


lums=[_np.asarray(Image.open(f'{out}/f_{i:05d}.jpg').convert('L'),dtype=_np.float32)/255 for i in range(1,n+1,max(1,n//12))]
L=_np.concatenate([x.ravel() for x in lums])
meta_l={'lumLo': round(float(_np.percentile(L,8)),3), 'lumHi': round(float(_np.percentile(L,99.5)),3)}
meta = {**meta_l, 'fps': a.fps, 'n': n, 'w': w, 'h': h, 'dur': n / a.fps, 'ext': 'jpg', 'src': os.path.basename(a.src), 'src_fps': pr.get('r_frame_rate')}
if a.pose:
    import mediapipe as mp, cv2
    pose = mp.solutions.pose.Pose(static_image_mode=False, model_complexity=1)
    P = []
    for i in range(1, n+1):
        im = cv2.cvtColor(cv2.imread(f'{out}/f_{i:05d}.jpg'), cv2.COLOR_BGR2RGB); r = pose.process(im)
        P.append([[round(l.x,4), round(l.y,4), round(l.visibility,3)] for l in r.pose_landmarks.landmark] if r.pose_landmarks else None)
    save_json(P, f'{out}/pose.json'); meta['pose'] = True
save_json(meta, f'{out}/meta.json', indent=1); print(meta)

#!/usr/bin/env bash
# Scaffold a kinetic music-video project in the CURRENT directory.
# usage: bash <skill>/scripts/scaffold.sh /path/to/master.mp3
# Needs: node >= 20, python3.11 via uv, ffmpeg/ffprobe, Google Chrome (Playwright drives it headless with GPU).
# macOS: Xcode command-line tools (swiftc) to build the Apple Vision matte tool. Elsewhere mattes come from another source.
set -euo pipefail
SKILL="$(cd "$(dirname "$0")/.." && pwd)"
MASTER_SRC="${1:?give the path to the song master}"
mkdir -p audio engine/scenes engine/data assets/fonts assets/refs assets/gen assets/clips assets/video assets/audio_slices assets/brand analysis out tools
# the song may already sit at audio/master.<ext> (cp refuses identical files and set -e would abort)
[ "$MASTER_SRC" -ef "audio/master.${MASTER_SRC##*.}" ] || cp "$MASTER_SRC" "audio/master.${MASTER_SRC##*.}"
cp -R "$SKILL/assets/engine/." engine/
cp "$SKILL/scripts/"*.py "$SKILL/scripts/"*.mjs "$SKILL/scripts/"*.swift "$SKILL/scripts/prep_clip.sh" tools/
[ -f engine/timeline.js ] || cp "$SKILL/assets/timeline.template.js" engine/timeline.js
cp -n "$SKILL/assets/scene.template.js" engine/scenes/_template.js || true
cp -n "$SKILL/references/engine_api.md" ENGINE_API.md || true
cp -n "$SKILL/references/agent_brief_template.md" AGENTS_BRIEF.md || true

# subject matte tool (Apple Vision foreground instance mask, ~0.1 s/frame, free)
if [ "$(uname)" = "Darwin" ] && command -v swiftc >/dev/null; then
  [ -x tools/matte ] || swiftc -O tools/matte.swift -o tools/matte || echo "note: swiftc failed (an unaccepted Xcode license is the usual cause: sudo xcodebuild -license accept). Mattes: Scenario background removal + tools/matte_keyed.py, or tools/matte_fallback.py."
else
  echo "note: no swiftc/macOS, so tools/matte was not built. Provide mattes as assets/video/<clip>/m_#####.png another way."
fi

# fonts (OFL, from google/fonts). The engine's index.html declares these families. Add project fonts the same way.
B=https://raw.githubusercontent.com/google/fonts/main/ofl
for p in "archivo/Archivo%5Bwdth,wght%5D.ttf:Archivo-VF.ttf" "archivoblack/ArchivoBlack-Regular.ttf:ArchivoBlack.ttf" "anton/Anton-Regular.ttf:Anton.ttf" \
  "jetbrainsmono/JetBrainsMono%5Bwght%5D.ttf:JetBrainsMono-VF.ttf" "doto/Doto%5BROND,wght%5D.ttf:Doto-VF.ttf" "blackhansans/BlackHanSans-Regular.ttf:BlackHanSans.ttf" \
  "unbounded/Unbounded%5Bwght%5D.ttf:Unbounded-VF.ttf" "instrumentserif/InstrumentSerif-Italic.ttf:InstrumentSerif-Italic.ttf" "instrumentserif/InstrumentSerif-Regular.ttf:InstrumentSerif.ttf"; do
  [ -f "assets/fonts/${p##*:}" ] || curl -fsSL "$B/${p%%:*}" -o "assets/fonts/${p##*:}"
done

# JS deps
[ -f package.json ] || npm init -y >/dev/null
npm i -s three@0.170.0 playwright@1.49 opentype.js >/dev/null

# Python env (audio analysis, lyric alignment, footage prep, tracking, sync checks)
export UV_NATIVE_TLS=1 # uv ignores the system certificate store by default and fails behind a TLS-inspecting proxy
uv venv .venv -q --python 3.11
.venv/bin/python -m ensurepip >/dev/null 2>&1 || true
uv pip install -q --python .venv/bin/python faster-whisper librosa soundfile numpy scipy demucs torch torchaudio opencv-python "mediapipe==0.10.14" matplotlib pillow "git+https://github.com/CPJKU/beat_this.git"
echo "scaffolded. master: audio/master.${MASTER_SRC##*.}"
echo "next: .venv/bin/python tools/audio_analysis.py audio/master.${MASTER_SRC##*.}"

"""Contact sheet with time labels. usage: python tools/sheet.py out.jpg cols img1 img2 ... [--force]
Refuses to replace an existing out.jpg unless --force is given, and never writes over one of its inputs."""
import os, sys, re
from PIL import Image, ImageDraw, ImageFont
force = '--force' in sys.argv; argv = [a for a in sys.argv[1:] if a != '--force']
if len(argv) < 3: sys.exit(__doc__)
out, cols, files = argv[0], int(argv[1]), argv[2:]
if os.path.abspath(out) in {os.path.abspath(f) for f in files}: sys.exit(f'{out} is also an input; choose another output path')
if os.path.exists(out) and not force: sys.exit(f'{out} exists; pass --force to replace it')
ims = [Image.open(f).convert('RGB') for f in files]; w, h = ims[0].size; rows = (len(ims)+cols-1)//cols
S = Image.new('RGB', (w*cols, h*rows)); d = ImageDraw.Draw(S)
try: font = ImageFont.truetype('assets/fonts/JetBrainsMono-VF.ttf', max(14, h//18))
except OSError: font = None
for i, (im, f) in enumerate(zip(ims, files)):
    x, y = (i % cols)*w, (i//cols)*h; S.paste(im, (x, y)); m = re.search(r't_0*([\d.]+)\.png', f)
    lab = m.group(1) if m else f.split('/')[-1]; d.rectangle([x, y, x+len(lab)*h//28+16, y+h//14], fill=(0,0,0)); d.text((x+6, y+2), lab, fill=(255,200,80), font=font)
S.save(out, quality=88); print(out)

"""Synthesize an original percussion guide track from the extracted beat/kick/snare timings (no song audio).
usage: python tools/synth_beat.py start dur out.wav [--force]   (refuses to replace an existing out.wav without --force)"""
import os, sys, json, numpy as np, soundfile as sf


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


force = '--force' in sys.argv; argv = [a for a in sys.argv[1:] if a != '--force']
if len(argv) < 3: sys.exit(__doc__)
t0, dur, out = float(argv[0]), float(argv[1]), argv[2]
if os.path.exists(out) and not force: sys.exit(f'{out} exists; pass --force to replace it')
A = load_json('engine/data/audio.json'); sr = 44100; n = int(dur*sr); y = np.zeros(n)
def add(sig, t):
    i = int((t - t0)*sr)
    if 0 <= i < n: m = min(len(sig), n-i); y[i:i+m] += sig[:m]
tt = np.arange(int(0.25*sr))/sr
kick = np.sin(2*np.pi*(50 + 90*np.exp(-tt*35))*tt) * np.exp(-tt*14)
rng = np.random.default_rng(0); sn = rng.standard_normal(int(0.18*sr)); ts = np.arange(len(sn))/sr
snare = (sn*0.6 + np.sin(2*np.pi*190*ts)*0.5) * np.exp(-ts*22)
hat = rng.standard_normal(int(0.04*sr)); hat = np.diff(np.concatenate([[0],hat])) * np.exp(-np.arange(len(hat))/sr*120) * 0.25
bl = np.arange(int(0.06*sr))/sr; blip = np.sin(2*np.pi*1760*bl)*np.exp(-bl*60)*0.35
for b in A['beats']: add(hat, b)
for b in A['downbeats']: add(blip, b)
kicks = [k for k in A['kicks'] if t0-0.1 < k < t0+dur]; snares = [s for s in A['snares'] if t0-0.1 < s < t0+dur]
# keep only strong, beat-aligned hits (within 60ms of a beat or half-beat) to avoid noise
bs = np.array(A['beats']); half = np.sort(np.concatenate([bs, (bs[:-1]+bs[1:])/2]))
near = lambda x: np.min(np.abs(half - x)) < 0.06
for k in kicks:
    if near(k): add(kick*0.9, k)
for s in snares:
    if near(s): add(snare*0.5, s)
y = y / (np.abs(y).max() + 1e-9) * 0.9
sf.write(out, y.astype(np.float32), sr, subtype='PCM_16'); print(out, len(kicks), len(snares))

"""Per-frame tracking for a prepped clip -> assets/video/<clip>/track.json and meta.track=true.
Each frame: {"pose": [[x,y,vis]*33] | null (MediaPipe, video UV, y down), "box": [x0,y0,x1,y1] matte bbox (UV), "c": [cx,cy] matte centroid, "a": matte area 0..1, "top": [x,y] highest matte point}
usage: python tools/track.py <clip>"""
import sys, os, json, numpy as np, cv2


def load_json(path):
    with open(path) as f:
        return json.load(f)


def save_json(obj, path, **kw):
    with open(path, 'w') as f:
        json.dump(obj, f, **kw)


if len(sys.argv) < 2: sys.exit(__doc__)
clip = sys.argv[1]; d = f'assets/video/{clip}'; meta = load_json(f'{d}/meta.json')
import mediapipe as mp

pose = mp.solutions.pose.Pose(static_image_mode=False, model_complexity=1, min_detection_confidence=0.4)
out = []
for i in range(1, meta['n'] + 1):
    im = cv2.imread(f'{d}/f_{i:05d}.jpg'); h, w = im.shape[:2]
    r = pose.process(cv2.cvtColor(im, cv2.COLOR_BGR2RGB))
    fr = {'pose': [[round(l.x, 4), round(l.y, 4), round(l.visibility, 2)] for l in r.pose_landmarks.landmark] if r.pose_landmarks else None}
    mp_ = f'{d}/m_{i:05d}.png'
    if os.path.exists(mp_):
        m = cv2.imread(mp_, cv2.IMREAD_GRAYSCALE); m = cv2.resize(m, (w // 4, h // 4)) > 127
        ys, xs = np.nonzero(m)
        if len(xs):
            W4, H4 = m.shape[1], m.shape[0]
            fr['box'] = [round(xs.min() / W4, 4), round(ys.min() / H4, 4), round(xs.max() / W4, 4), round(ys.max() / H4, 4)]
            fr['c'] = [round(xs.mean() / W4, 4), round(ys.mean() / H4, 4)]; fr['a'] = round(len(xs) / m.size, 4)
            k = ys.argmin(); fr['top'] = [round(xs[k] / W4, 4), round(ys[k] / H4, 4)]
    out.append(fr)
save_json(out, f'{d}/track.json')
meta['track'] = True; save_json(meta, f'{d}/meta.json', indent=1)
print(clip, 'tracked', len(out), 'pose frames', sum(1 for f in out if f['pose']))