feat(whisper+captions): language detection, audio-reactive captions, multilingual defaults

CLAUDE.md

@@ -95,13 +95,13 @@ Default is `small.en`. Upgrade for better accuracy:
 
 | Model       | Size   | Use case                         |
 | ----------- | ------ | -------------------------------- |
-| `tiny.en`   | 75 MB  | Quick testing                    |
-| `base.en`   | 142 MB | Short clips, clear audio         |
-| `small.en`  | 466 MB | **Default** — most content       |
-| `medium.en` | 1.5 GB | Important content, noisy audio   |
-| `large-v3`  | 3.1 GB | Multilingual, production quality |
+| `tiny`      | 75 MB  | Quick testing                    |
+| `base`      | 142 MB | Short clips, clear audio         |
+| `small`     | 466 MB | **Default** — most content       |
+| `medium`    | 1.5 GB | Important content, noisy audio   |
+| `large-v3`  | 3.1 GB | Production quality               |
 
-Use `.en` suffix for English-only (more accurate). Drop it for multilingual content.
+**Only use `.en` suffix when you know the audio is English.** `.en` models translate non-English audio into English instead of transcribing it.
 
 ### Supported transcript formats
 

packages/cli/src/commands/transcribe.ts

@@ -103,7 +103,8 @@ async function transcribeAudio(
   opts: { model?: string; language?: string; json?: boolean },
 ): Promise<void> {
   const { transcribe } = await import("../whisper/transcribe.js");
-  const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");
+  const { loadTranscript, patchCaptionHtml, stripBeforeOnset } =
+    await import("../whisper/normalize.js");
 
   const model = opts.model ?? DEFAULT_MODEL;
   const spin = opts.json ? null : clack.spinner();
@@ -116,7 +117,19 @@ async function transcribeAudio(
       onProgress: spin ? (msg) => spin.message(msg) : undefined,
     });
 
-    const { words } = loadTranscript(result.transcriptPath);
+    let { words } = loadTranscript(result.transcriptPath);
+
+    if (result.speechOnsetSeconds != null) {
+      const before = words.length;
+      words = stripBeforeOnset(words, result.speechOnsetSeconds);
+      const stripped = before - words.length;
+      if (stripped > 0 && !opts.json) {
+        spin?.message(
+          `Stripped ${stripped} words before speech onset at ${result.speechOnsetSeconds.toFixed(1)}s`,
+        );
+      }
+    }
+
     writeFileSync(result.transcriptPath, JSON.stringify(words, null, 2));
     patchCaptionHtml(dir, words);
 
@@ -127,13 +140,18 @@ async function transcribeAudio(
           model,
           wordCount: words.length,
           durationSeconds: result.durationSeconds,
+          speechOnsetSeconds: result.speechOnsetSeconds,
           transcriptPath: result.transcriptPath,
         }),
       );
     } else {
-      spin!.stop(
+      const onsetNote =
+        result.speechOnsetSeconds != null
+          ? ` — speech detected at ${result.speechOnsetSeconds.toFixed(1)}s`
+          : "";
+      spin?.stop(
         c.success(
-          `Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s)`,
+          `Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s${onsetNote})`,
         ),
       );
     }
@@ -142,7 +160,7 @@ async function transcribeAudio(
     if (opts.json) {
       console.log(JSON.stringify({ ok: false, error: message }));
     } else {
-      spin!.stop(c.error(`Transcription failed: ${message}`));
+      spin?.stop(c.error(`Transcription failed: ${message}`));
     }
     process.exit(1);
   }

packages/cli/src/whisper/normalize.test.ts

@@ -2,7 +2,8 @@ import { describe, it, expect, afterEach } from "vitest";
 import { writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
-import { loadTranscript, detectFormat, patchCaptionHtml } from "./normalize.js";
+import { loadTranscript, detectFormat, patchCaptionHtml, stripBeforeOnset } from "./normalize.js";
+import { detectSpeechOnset } from "./transcribe.js";
 
 function tmpFile(name: string, content: string): string {
   const dir = join(tmpdir(), `hf-normalize-test-${Date.now()}`);
@@ -117,7 +118,7 @@ describe("loadTranscript", () => {
     );
     const { words } = loadTranscript(path);
     expect(words).toHaveLength(1);
-    expect(words[0]!.text).toBe("Hello");
+    expect(words[0]?.text).toBe("Hello");
   });
 
   it("parses OpenAI Whisper API response", () => {
@@ -183,8 +184,8 @@ Short format
 `;
     const path = tmpFile("short.vtt", vtt);
     const { words } = loadTranscript(path);
-    expect(words[0]!.start).toBeCloseTo(83.456, 2);
-    expect(words[0]!.end).toBe(120.0);
+    expect(words[0]?.start).toBeCloseTo(83.456, 2);
+    expect(words[0]?.end).toBe(120.0);
   });
 
   it("strips HTML tags from SRT/VTT", () => {
@@ -194,7 +195,7 @@ Short format
 `;
     const path = tmpFile("tags.srt", srt);
     const { words } = loadTranscript(path);
-    expect(words[0]!.text).toBe("Bold and italic");
+    expect(words[0]?.text).toBe("Bold and italic");
   });
 
   it("passes through normalized word arrays", () => {
@@ -209,6 +210,172 @@ Short format
   });
 });
 
+describe("whisper-cpp contraction merging", () => {
+  it("merges didn + 't into didn't", () => {
+    const path = tmpFile(
+      "contractions.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              { text: " I", offsets: { from: 0, to: 200 } },
+              { text: " didn", offsets: { from: 200, to: 500 } },
+              { text: "'t", offsets: { from: 500, to: 700 } },
+              { text: " know", offsets: { from: 700, to: 1000 } },
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    expect(words).toEqual([
+      { text: "I", start: 0, end: 0.2 },
+      { text: "didn't", start: 0.2, end: 0.7 },
+      { text: "know", start: 0.7, end: 1 },
+    ]);
+  });
+
+  it("merges I + 'm into I'm", () => {
+    const path = tmpFile(
+      "im.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              { text: " I", offsets: { from: 0, to: 100 } },
+              { text: "'m", offsets: { from: 100, to: 300 } },
+              { text: " done", offsets: { from: 300, to: 600 } },
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    expect(words[0]?.text).toBe("I'm");
+    expect(words[0]?.end).toBe(0.3);
+  });
+
+  it("merges could + 've into could've", () => {
+    const path = tmpFile(
+      "couldve.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              { text: " could", offsets: { from: 0, to: 400 } },
+              { text: "'ve", offsets: { from: 400, to: 600 } },
+              { text: " been", offsets: { from: 600, to: 900 } },
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    expect(words[0]?.text).toBe("could've");
+  });
+});
+
+describe("whisper-cpp fragment merging", () => {
+  it("merges single capital + lowercase: C + aught -> Caught", () => {
+    const path = tmpFile(
+      "fragments.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              { text: " C", offsets: { from: 0, to: 100 } },
+              { text: "aught", offsets: { from: 100, to: 500 } },
+              { text: " a", offsets: { from: 500, to: 600 } },
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    expect(words[0]?.text).toBe("Caught");
+    expect(words[0]?.end).toBe(0.5);
+    expect(words).toHaveLength(2);
+  });
+
+  it("merges consonant + in': shin + in' -> shinin'", () => {
+    const path = tmpFile(
+      "dropg.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              { text: " shin", offsets: { from: 0, to: 300 } },
+              { text: "in'", offsets: { from: 300, to: 500 } },
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    expect(words).toHaveLength(1);
+    expect(words[0]?.text).toBe("shinin'");
+  });
+});
+
+describe("whisper-cpp zero-duration interpolation", () => {
+  it("interpolates a cluster of zero-duration words", () => {
+    const path = tmpFile(
+      "zerodur.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              { text: " hello", offsets: { from: 0, to: 500 } },
+              { text: " we", offsets: { from: 1000, to: 1000 } },
+              { text: " are", offsets: { from: 1000, to: 1000 } },
+              { text: " here", offsets: { from: 1000, to: 1000 } },
+              { text: " now", offsets: { from: 1500, to: 2000 } },
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    expect(words).toHaveLength(5);
+    // The three zero-duration words should be spread between 0.5 and 1.5
+    const we = words[1] ?? { start: 0, end: 0, text: "" };
+    const are = words[2] ?? { start: 0, end: 0, text: "" };
+    const here = words[3] ?? { start: 0, end: 0, text: "" };
+    expect(we.start).toBeCloseTo(0.5, 1);
+    expect(we.end).toBeCloseTo(0.833, 1);
+    expect(are.start).toBeCloseTo(0.833, 1);
+    expect(are.end).toBeCloseTo(1.167, 1);
+    expect(here.start).toBeCloseTo(1.167, 1);
+    expect(here.end).toBeCloseTo(1.5, 1);
+    // Each should have positive duration
+    expect(we.end).toBeGreaterThan(we.start);
+    expect(are.end).toBeGreaterThan(are.start);
+    expect(here.end).toBeGreaterThan(here.start);
+  });
+
+  it("handles isolated zero-duration word", () => {
+    const path = tmpFile(
+      "singlezero.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              { text: " hello", offsets: { from: 0, to: 500 } },
+              { text: " I", offsets: { from: 800, to: 800 } },
+              { text: " know", offsets: { from: 1000, to: 1500 } },
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    const iWord = words[1] ?? { start: 0, end: 0, text: "" };
+    expect(iWord.end).toBeGreaterThan(iWord.start);
+    expect(iWord.start).toBeCloseTo(0.5, 1);
+    expect(iWord.end).toBeCloseTo(1, 1);
+  });
+});
+
 describe("patchCaptionHtml", () => {
   it("replaces const script = [] in HTML files", () => {
     const dir = join(tmpdir(), `hf-patch-test-${Date.now()}`);
@@ -276,3 +443,105 @@ describe("patchCaptionHtml", () => {
     expect(result).toBe(html);
   });
 });
+
+describe("detectSpeechOnset", () => {
+  function makeSyntheticWav(
+    sampleRate: number,
+    durationSeconds: number,
+    energyFn: (t: number) => number,
+  ): string {
+    const numSamples = Math.floor(sampleRate * durationSeconds);
+    const dataSize = numSamples * 2;
+    const buf = Buffer.alloc(44 + dataSize);
+    // RIFF header
+    buf.write("RIFF", 0);
+    buf.writeUInt32LE(36 + dataSize, 4);
+    buf.write("WAVE", 8);
+    buf.write("fmt ", 12);
+    buf.writeUInt32LE(16, 16); // chunk size
+    buf.writeUInt16LE(1, 20); // PCM
+    buf.writeUInt16LE(1, 22); // mono
+    buf.writeUInt32LE(sampleRate, 24);
+    buf.writeUInt32LE(sampleRate * 2, 28); // byte rate
+    buf.writeUInt16LE(2, 32); // block align
+    buf.writeUInt16LE(16, 34); // bits per sample
+    buf.write("data", 36);
+    buf.writeUInt32LE(dataSize, 40);
+    for (let i = 0; i < numSamples; i++) {
+      const t = i / sampleRate;
+      const amplitude = energyFn(t);
+      buf.writeInt16LE(Math.round(amplitude * 32767), 44 + i * 2);
+    }
+    const path = join(tmpdir(), `hf-wav-test-${Date.now()}-${Math.floor(Math.random() * 1e6)}.wav`);
+    writeFileSync(path, buf);
+    dirs.push(path);
+    return path;
+  }
+
+  it("detects onset when silence transitions to loud", () => {
+    const wavPath = makeSyntheticWav(16000, 15, (t) => (t < 5 ? 0.01 : 0.8));
+    const onset = detectSpeechOnset(wavPath);
+    expect(onset).not.toBeNull();
+    expect(onset!).toBeGreaterThanOrEqual(4);
+    expect(onset!).toBeLessThanOrEqual(7);
+  });
+
+  it("returns null for consistent energy throughout", () => {
+    const wavPath = makeSyntheticWav(16000, 10, () => 0.5);
+    const onset = detectSpeechOnset(wavPath);
+    expect(onset).toBeNull();
+  });
+
+  it("returns null for very short audio", () => {
+    const wavPath = makeSyntheticWav(16000, 2, () => 0.5);
+    const onset = detectSpeechOnset(wavPath);
+    expect(onset).toBeNull();
+  });
+
+  it("returns null when onset is too early (< 3s)", () => {
+    const wavPath = makeSyntheticWav(16000, 10, (t) => (t < 1 ? 0.01 : 0.8));
+    const onset = detectSpeechOnset(wavPath);
+    expect(onset).toBeNull();
+  });
+});
+
+describe("stripBeforeOnset", () => {
+  it("removes words before onset time", () => {
+    const words = [
+      { text: "ghost", start: 0.5, end: 2.0 },
+      { text: "alone", start: 3.0, end: 5.0 },
+      { text: "Given", start: 19.0, end: 19.5 },
+      { text: "the", start: 19.5, end: 20.0 },
+    ];
+    const result = stripBeforeOnset(words, 18.5);
+    expect(result).toHaveLength(2);
+    expect(result[0]!.text).toBe("Given");
+  });
+
+  it("keeps words within 0.5s tolerance of onset", () => {
+    const words = [
+      { text: "hello", start: 18.2, end: 18.8 },
+      { text: "world", start: 19.0, end: 19.5 },
+    ];
+    const result = stripBeforeOnset(words, 18.5);
+    expect(result).toHaveLength(2);
+  });
+
+  it("keeps everything when onset is 0", () => {
+    const words = [
+      { text: "hello", start: 0.1, end: 0.5 },
+      { text: "world", start: 0.6, end: 1.0 },
+    ];
+    const result = stripBeforeOnset(words, 0);
+    expect(result).toHaveLength(2);
+  });
+
+  it("returns empty array when all words are before onset", () => {
+    const words = [
+      { text: "ghost", start: 0.5, end: 2.0 },
+      { text: "alone", start: 3.0, end: 5.0 },
+    ];
+    const result = stripBeforeOnset(words, 20.0);
+    expect(result).toHaveLength(0);
+  });
+});