Hook-timing diagnostics, WhisperX re-alignment, and merge-doc token fixes

CLAUDE.md

@@ -174,7 +174,7 @@ ty    = (0.5 - vp.cy) × 100%
 | `PAUSE_THRESHOLD` | 0.8 s | `edit-transcript.js` |
 | `WORD_DURATION_ESTIMATE` | 0.4 s | `edit-transcript.js` |
 | `CUT_START_BIAS` | 1.0 | `edit-transcript.js` |
-| `HOOK_TAIL_PAD_UNBOUNDED_SECONDS` | 0.16 s | `remotion/lib/hookTiming.ts` |
+| `HOOK_TAIL_PAD_UNBOUNDED_SECONDS` | 0.50 s | `remotion/lib/hookTiming.ts` |
 | `HOOK_TAIL_PAD_BOUNDED_SECONDS` | 0.02 s | `remotion/lib/hookTiming.ts` |
 | `HOOK_BRIDGE_MAX_GAP_SECONDS` | 1.0 s | `remotion/lib/hookTiming.ts` |
 | `HOOK_END_FADE_FRAMES` | 12 | `SegmentPlayer.tsx` |

package.json

@@ -16,14 +16,15 @@
     "cut:preview": "node scripts/cut-preview.js",
     "dev": "next dev",
     "diarize": "node scripts/diarize/diarize-audio.js",
+    "diagnose:hooks": "tsx scripts/diagnostics/verify-hook-timing.ts",
     "lint": "eslint",
     "prepare": "husky || true",
     "remotion:gallery": "npx remotion studio remotion/galleryRoot.ts",
     "remotion:studio": "npx remotion studio",
     "render:episode": "node scripts/render-episode.js",
     "render:episode:resume": "node scripts/render-episode-resumable.js",
     "render:episode:chunk": "node scripts/render-episode-resumable.js",
-    "render:hook-intro": "node scripts/render-hook-intro.js",
+    "render:hook-intro": "tsx scripts/render-hook-intro.js",
     "check:media-urls": "node scripts/lib/checkMediaUrls.js",
     "review:guest": "node scripts/guest-review.js",
     "shorts:extract-doc": "node scripts/shorts/extract-short-doc.js",

scripts/align/align-transcript.js

@@ -7,8 +7,6 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import { stampMetadata } from '../config/metadata.js';
 
-const ALIGN_SCRIPT_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'run_whisperx_align.py');
-
 function parseArgs() {
   const args = process.argv.slice(2);
   const result = {};
@@ -41,7 +39,7 @@ async function autoDetectFile(dir, extensions) {
   return match ? path.join(dir, match) : null;
 }
 
-function spawnPython(pythonBin, args) {
+export function spawnPython(pythonBin, args) {
   return new Promise((resolve, reject) => {
     const proc = spawn(pythonBin, args, {
       stdio: 'inherit',
@@ -161,12 +159,19 @@ function assignTokenTimes(tokens, alignedWords, oldStart, oldEnd, newStart, newE
     if (alignedEnd !== undefined) {
       // Clamp to [t_dtw, newEnd] so t_end is always a valid, monotonically sound boundary.
       result.t_end = Number(Math.min(newEnd, Math.max(t, alignedEnd)).toFixed(3));
+    } else {
+      // Unmatched token (fell back to a remapped/interpolated t_dtw): its old
+      // t_end belonged to the old t_dtw and can end up before the new one
+      // (invalid). Clear it — hookClipEnd()/buildCaptions() already treat a
+      // missing t_end as "unknown", which is honest here; a stale wrong value
+      // is not.
+      delete result.t_end;
     }
     return result;
   });
 }
 
-function applyAlignment(rawTranscript, alignedPayload) {
+export function applyAlignment(rawTranscript, alignedPayload) {
   const alignedByRawIndex = new Map(
     (alignedPayload?.segments || []).map((seg) => [seg.raw_index, seg])
   );
@@ -250,6 +255,7 @@ async function main() {
   const { audioPath, rawPath, pythonBin, language, device } = await resolveArgs(cwd);
 
   const tempOutputPath = path.join(os.tmpdir(), `deckcreate-alignment-${Date.now()}.json`);
+  const alignScriptPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'run_whisperx_align.py');
 
   console.log('\nForced Alignment');
   console.log(`  Audio:      ${audioPath}`);
@@ -260,7 +266,7 @@ async function main() {
 
   try {
     await spawnPython(pythonBin, [
-      ALIGN_SCRIPT_PATH,
+      alignScriptPath,
       '--audio', audioPath,
       '--raw', rawPath,
       '--out', tempOutputPath,

scripts/align/realign-hooks.js

@@ -0,0 +1,311 @@
+#!/usr/bin/env tsx
+/**
+ * Scoped WhisperX re-alignment for hook segments whose token timestamps are
+ * compressed/unreliable relative to their (verified-by-ear) hookFrom/hookTo.
+ *
+ * Why this exists: HookOverlay's buildCaptions() filters a segment's tokens to
+ * those whose t_dtw falls within [hookFrom ?? start, hookClipEnd(...)) — if a
+ * segment's original WhisperX alignment placed all its tokens outside that
+ * window (confirmed on this transcript for several hooks), captions render as
+ * empty for the whole hook, or partially wrong. Re-running the *same* WhisperX
+ * forced-alignment primitive the pipeline already uses (see
+ * run_whisperx_align.py), but scoped to just the affected hooks and given a
+ * tighter/more accurate search window (their real hookFrom/hookTo instead of
+ * the original segment's own start/end), lets WhisperX re-place the tokens
+ * correctly — without needing to re-run alignment on the whole episode, and
+ * without touching segments whose *text* doesn't match the audio at all (no
+ * re-alignment can fix that; those must stay manually excluded via --skip).
+ *
+ * Merge strategy is intentionally NOT a reuse of align-transcript.js's
+ * assignTokenTimes()/applyAlignment(): those assume the aligned text covers
+ * every token in the segment, then proportionally remap any unmatched token
+ * from its *old* position — correct for whole-episode re-alignment, where the
+ * segment's own start/end is the alignment window. Here the alignment target
+ * is often just a hook's hookPhrase (a SUBSET of the segment's words, e.g.
+ * "loop engineering" out of "is called loop engineering."), so tokens for
+ * words outside the phrase are *expected* to go unmatched — remapping them
+ * from an old position that may itself already be wrong compounds errors on
+ * every re-run (confirmed: a second run collapsed all tokens to the window's
+ * end because the "old" reference position had already moved). Since those
+ * words are outside the hook's own clip window anyway, their exact timestamp
+ * doesn't matter for captions — only that they stay excluded — so unmatched
+ * tokens are deterministically pushed to sourceStart-1 instead of remapped.
+ *
+ * Usage:
+ *   npx tsx scripts/align/realign-hooks.js [options]
+ *
+ * Options:
+ *   --transcript <path>   transcript.json path (default: public/edit/transcript.json)
+ *   --audio <path>        Audio file matching the transcript's timeline
+ *                          (default: public/transcribe/input/audio.wav)
+ *   --python <path>       Python binary (default: python3)
+ *   --device <device>     cpu|cuda|mps|auto (default: auto)
+ *   --language <lang>     (default: en)
+ *   --skip <indices>      Comma-separated hook indices to exclude (segments whose
+ *                          text doesn't match the audio — re-alignment can't fix that)
+ *   --dry-run             Print what would be aligned without calling WhisperX
+ */
+
+import fs from 'fs-extra';
+import os from 'os';
+import path from 'path';
+import { hookClipEnd } from '../../remotion/lib/hookTiming';
+import { spawnPython } from './align-transcript.js';
+import { stampMetadata } from '../config/metadata.js';
+
+const ALIGN_SCRIPT_PATH = new URL('./run_whisperx_align.py', import.meta.url).pathname;
+
+function parseArgs(argv) {
+  const out = {
+    transcriptPath: path.join('public', 'edit', 'transcript.json'),
+    audioPath: path.join('public', 'transcribe', 'input', 'audio.wav'),
+    pythonBin: 'python3',
+    device: 'auto',
+    language: 'en',
+    skip: [],
+    dryRun: false,
+    help: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--transcript' && argv[i + 1]) out.transcriptPath = argv[++i];
+    else if (a === '--audio' && argv[i + 1]) out.audioPath = argv[++i];
+    else if (a === '--python' && argv[i + 1]) out.pythonBin = argv[++i];
+    else if (a === '--device' && argv[i + 1]) out.device = argv[++i];
+    else if (a === '--language' && argv[i + 1]) out.language = argv[++i];
+    else if (a === '--skip' && argv[i + 1]) out.skip = argv[++i].split(',').map(s => Number(s.trim()));
+    else if (a === '--dry-run') out.dryRun = true;
+    else if (a === '--help' || a === '-h') out.help = true;
+  }
+  return out;
+}
+
+function printHelp() {
+  console.log(`
+Scoped WhisperX re-alignment for hook segments
+
+Re-runs forced alignment for hook segments, scoped to each hook's own
+[hookFrom, hookClipEnd) window, to fix compressed/wrong token timestamps that
+cause missing or inaccurate hook captions. Skips segments whose text doesn't
+match the audio at all (pass their index via --skip; re-alignment cannot fix
+a content mismatch, only a timing one).
+
+Usage:
+  npx tsx scripts/align/realign-hooks.js [options]
+
+Options:
+  --transcript <path>   transcript.json path (default: public/edit/transcript.json)
+  --audio <path>        Audio file matching the transcript's timeline
+                         (default: public/transcribe/input/audio.wav)
+  --python <path>       Python binary (default: python3)
+  --device <device>     cpu|cuda|mps|auto (default: auto)
+  --language <lang>     (default: en)
+  --skip <indices>      Comma-separated hook indices to exclude
+  --dry-run             Print what would be aligned without calling WhisperX
+  --help, -h             Show this help
+`);
+}
+
+function normalizeWord(text) {
+  return (text || '').trim().replace(/^[^\w']+|[^\w']+$/g, '').toLowerCase();
+}
+
+function isSpecialToken(token) {
+  return /_[A-Z]+_/.test((token?.text || '').trim());
+}
+
+/**
+ * Groups raw (possibly BPE sub-word) tokens using the leading-space
+ * heuristic: a token with a leading space (or the first token) starts a new
+ * word group; a token without one is a continuation of the previous group
+ * (handles contractions like "'t" too, since those never have a leading
+ * space either). Punctuation/empty/special tokens are never grouped with
+ * neighbors — they always stand alone, since they carry no letters to concat
+ * and don't need real timing (buildCaptions groups them onto an adjacent
+ * real word by position, not by their own timestamp).
+ */
+function groupTokensForMatching(tokens) {
+  const groups = [];
+  tokens.forEach((token, idx) => {
+    const hasLetters = !!normalizeWord(token.text);
+    if (!hasLetters || isSpecialToken(token)) {
+      groups.push({ indices: [idx], text: '' });
+      return;
+    }
+    const prev = groups[groups.length - 1];
+    if (prev && prev.text && !token.text.startsWith(' ')) {
+      prev.indices.push(idx);
+      prev.text += token.text;
+    } else {
+      groups.push({ indices: [idx], text: token.text });
+    }
+  });
+  return groups;
+}
+
+/**
+ * Matches a segment's existing tokens against WhisperX's returned words.
+ * Whisper's own tokens are frequently BPE sub-word split (e.g. "orchest" +
+ * "rate" for "orchestrate", "Comp" + "ounding" for "Compounding") while
+ * WhisperX's alignment output is whole-word — comparing each raw token
+ * individually against whole words means neither half of a split word ever
+ * matches, silently dropping it entirely (confirmed: this is exactly why
+ * "orchestrate" and "Compounding" went missing from captions). Tokens are
+ * grouped into words first (groupTokensForMatching), matched as whole words,
+ * then a matched word's [start, end] is divided evenly across its
+ * constituent sub-tokens so every raw token still gets a valid, monotonic
+ * timestamp.
+ *
+ * Matched words are clamped to start no earlier than sourceStart. WhisperX's
+ * align() consistently returns a handful of leading words ~0.5s before the
+ * given window start (confirmed across many hooks on this transcript — an
+ * internal context-padding behavior, not natural speech variance: identical
+ * 0.500s across completely different words/positions). Left unclamped, that
+ * pushes a word's t_dtw before sourceStart, which either drops it from
+ * HookOverlay's caption filter entirely (t_dtw < sourceStart requires a
+ * defined t_end > sourceStart to survive via the "early-start overlap"
+ * fallback — many WhisperX words don't have one) or, worse, lets it survive
+ * via that fallback while a *different* word in the same phrase without a
+ * t_end doesn't, silently concatenating adjacent words with no space
+ * (observed as "these buzzwords" rendering as "thesewords" once "buzz" got
+ * dropped but its no-leading-space continuation "words" didn't). sourceStart
+ * is the hook's own verified boundary — a matched word belongs inside it by
+ * definition, so clamping forward is always correct here, never a truncation.
+ *
+ * Unmatched groups (punctuation, special markers, or a real word that isn't
+ * part of the aligned phrase) are pushed to sourceStart-1 — a deterministic
+ * placeholder that guarantees exclusion from any [sourceStart, sourceEnd)
+ * window filter, since its own timestamp is meaningless outside that phrase.
+ */
+function mergeAlignedWords(tokens, alignedWords, sourceStart) {
+  const words = alignedWords.map(w => ({ ...w, normalized: normalizeWord(w.word) }));
+  const groups = groupTokensForMatching(tokens);
+  const placeholder = sourceStart - 1;
+  const results = new Array(tokens.length);
+  let searchStart = 0;
+
+  for (const group of groups) {
+    const normalized = normalizeWord(group.text);
+    let matchIndex = -1;
+    if (normalized) {
+      for (let wi = searchStart; wi < words.length; wi++) {
+        if (words[wi].normalized && words[wi].normalized === normalized) {
+          matchIndex = wi;
+          break;
+        }
+      }
+    }
+
+    if (matchIndex >= 0) {
+      searchStart = matchIndex + 1;
+      const w = words[matchIndex];
+      const start = Math.max(w.start, sourceStart);
+      const end = Math.max(w.end, start + 0.01);
+      const step = (end - start) / group.indices.length;
+      group.indices.forEach((tokenIdx, i) => {
+        results[tokenIdx] = { ...tokens[tokenIdx], t_dtw: start + step * i, t_end: start + step * (i + 1) };
+      });
+    } else {
+      group.indices.forEach(tokenIdx => {
+        const result = { ...tokens[tokenIdx], t_dtw: placeholder };
+        delete result.t_end;
+        results[tokenIdx] = result;
+      });
+    }
+  }
+
+  return results;
+}
+
+function buildTargets(hookSegments, skipSet) {
+  const alignInputs = [];
+  const targets = [];
+
+  for (let i = 0; i < hookSegments.length; i++) {
+    if (skipSet.has(i)) continue;
+    const seg = hookSegments[i];
+    const next = hookSegments[i + 1];
+    const nextHookStart = next ? (next.hookFrom ?? next.start) : undefined;
+    const sourceStart = seg.hookFrom ?? seg.start;
+    const sourceEnd = hookClipEnd(seg, nextHookStart);
+    const text = seg.hookPhrase || seg.text;
+
+    if (!text || !text.trim()) continue;
+
+    alignInputs.push({ text, start: sourceStart, end: Math.max(sourceEnd, sourceStart + 0.01) });
+    targets.push({ hookIndex: i, segment: seg, sourceStart });
+  }
+
+  return { alignInputs, targets };
+}
+
+async function main() {
+  const cli = parseArgs(process.argv.slice(2));
+  if (cli.help) { printHelp(); return; }
+
+  const cwd = process.cwd();
+  const transcriptPath = path.resolve(cwd, cli.transcriptPath);
+  const audioPath = path.resolve(cwd, cli.audioPath);
+
+  if (!await fs.pathExists(transcriptPath)) throw new Error(`Transcript not found: ${transcriptPath}`);
+  if (!cli.dryRun && !await fs.pathExists(audioPath)) throw new Error(`Audio not found: ${audioPath}`);
+
+  const transcript = await fs.readJson(transcriptPath);
+  const hookSegments = (transcript.segments || []).filter(s => s.hook && !s.cut);
+  const skipSet = new Set(cli.skip);
+
+  const { alignInputs, targets } = buildTargets(hookSegments, skipSet);
+
+  console.log(`\n[realign-hooks] ${targets.length}/${hookSegments.length} hook segments targeted `
+    + `(${skipSet.size} skipped: ${cli.skip.join(', ') || 'none'})\n`);
+
+  if (cli.dryRun) {
+    targets.forEach((t, i) => {
+      console.log(`#${t.hookIndex}  [${alignInputs[i].start.toFixed(3)}, ${alignInputs[i].end.toFixed(3)}]  "${alignInputs[i].text}"`);
+    });
+    return;
+  }
+
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'realign-hooks-'));
+  const scopedRawPath = path.join(tmpDir, 'scoped-raw.json');
+  const scopedOutPath = path.join(tmpDir, 'scoped-aligned.json');
+
+  try {
+    await fs.writeJson(scopedRawPath, { segments: alignInputs });
+
+    console.log('[realign-hooks] Running WhisperX forced alignment...\n');
+    await spawnPython(cli.pythonBin, [
+      ALIGN_SCRIPT_PATH,
+      '--audio', audioPath,
+      '--raw', scopedRawPath,
+      '--out', scopedOutPath,
+      '--device', cli.device,
+      '--language', cli.language,
+    ]);
+
+    const alignedPayload = await fs.readJson(scopedOutPath);
+    const alignedByRawIndex = new Map((alignedPayload.segments || []).map(seg => [seg.raw_index, seg]));
+
+    let changed = 0;
+    targets.forEach((target, i) => {
+      const aligned = alignedByRawIndex.get(i);
+      if (!aligned) {
+        console.log(`  #${target.hookIndex}: no alignment result — left untouched`);
+        return;
+      }
+      target.segment.tokens = mergeAlignedWords(target.segment.tokens, aligned.words || [], target.sourceStart);
+      changed++;
+    });
+
+    await fs.writeJson(transcriptPath, stampMetadata(transcript), { spaces: 2 });
+
+    console.log(`\n[realign-hooks] Updated tokens for ${changed}/${targets.length} hook(s) in ${transcriptPath}`);
+  } finally {
+    await fs.remove(tmpDir).catch(() => {});
+  }
+}
+
+main().catch(err => {
+  console.error(`\n[realign-hooks] Error: ${err.message}`);
+  process.exit(1);
+});