diff --git a/frontend/src/audio/mic-stream.ts b/frontend/src/audio/mic-stream.ts index bc190a0..18be4b2 100644 --- a/frontend/src/audio/mic-stream.ts +++ b/frontend/src/audio/mic-stream.ts @@ -4,6 +4,10 @@ import { TRANSCRIBE_SAMPLE_RATE_HZ } from "../api/transcribe.js"; // `addModule` rejects) — so it can live next to this file instead of `public/`. import rawPcm16ProcessorUrl from "./rawPcm16Processor.js?url&no-inline"; +const CHUNK_DURATION_MS = 100; +const CHUNK_SAMPLES = + (TRANSCRIBE_SAMPLE_RATE_HZ * CHUNK_DURATION_MS) / 1000; + // #region microphone-stream export async function startMicrophoneStream( onChunk: (chunk: Int16Array) => void, @@ -27,32 +31,13 @@ export async function startMicrophoneStream( await audioCtx.audioWorklet.addModule(rawPcm16ProcessorUrl); const mediaStreamSource = audioCtx.createMediaStreamSource(stream); - const worklet = new AudioWorkletNode(audioCtx, "rawPcm16Processor"); - - const CHUNK_SAMPLES = TRANSCRIBE_SAMPLE_RATE_HZ / 10; // 100 ms of audio - let chunkBuffer = new Int16Array(CHUNK_SAMPLES); - let chunkBufferCurrentLength = 0; + const worklet = new AudioWorkletNode(audioCtx, "rawPcm16Processor", { + processorOptions: { chunkSamples: CHUNK_SAMPLES }, + }); - worklet.port.onmessage = ({ - data: incomingSamples, - }: MessageEvent) => { - let alreadyAdded = 0; - while (alreadyAdded < incomingSamples.length) { - const samplesToTake = Math.min( - CHUNK_SAMPLES - chunkBufferCurrentLength, // The remaining space in the buffer - incomingSamples.length - alreadyAdded, // The remaining samples in the incoming array - ); - for (let i = 0; i < samplesToTake; i++) { - chunkBuffer[chunkBufferCurrentLength + i] = incomingSamples[alreadyAdded + i]; - } - chunkBufferCurrentLength += samplesToTake; - alreadyAdded += samplesToTake; - if (chunkBufferCurrentLength === CHUNK_SAMPLES) { - onChunk(chunkBuffer); - chunkBuffer = new Int16Array(CHUNK_SAMPLES); - chunkBufferCurrentLength = 0; - } - } + // The worklet converts and chunks on the audio thread; the main thread just forwards. + worklet.port.onmessage = ({ data }: MessageEvent) => { + onChunk(data); }; mediaStreamSource.connect(worklet); diff --git a/frontend/src/audio/rawPcm16Processor.js b/frontend/src/audio/rawPcm16Processor.js index 0bf960b..a610fb9 100644 --- a/frontend/src/audio/rawPcm16Processor.js +++ b/frontend/src/audio/rawPcm16Processor.js @@ -1,20 +1,46 @@ // #region pcm16-worklet -// An AudioWorklet that converts the browser's Float32 mic samples to PCM_S16LE — -// the encoding the Nabla streaming APIs require. Loaded via audioWorklet.addModule(). + +// An AudioWorklet that converts the browser's Float32 mic samples to PCM_S16LE and +// emits fixed-size chunks — the encoding and pacing the Nabla streaming APIs require. +// Loaded via audioWorklet.addModule(). class RawPcm16Processor extends AudioWorkletProcessor { + constructor(options) { + super(); + const chunkSamples = options.processorOptions?.chunkSamples ?? 1600; + this.accumulator = new Pcm16ChunkAccumulator(this.port, chunkSamples); + } + process(inputs) { const input = inputs[0]?.[0]; if (!input) { return true; } - const pcm16 = new Int16Array(input.length); + this.accumulator.appendFloat32(input); + return true; + } +} + +// Buffers converted PCM samples until a full chunk is ready, then posts it to the +// main thread. Lives entirely on the audio rendering thread. +class Pcm16ChunkAccumulator { + constructor(port, chunkSamples) { + this.port = port; + this.chunkSamples = chunkSamples; + this.buffer = new Int16Array(chunkSamples); + this.length = 0; + } + + appendFloat32(input) { for (let i = 0; i < input.length; i++) { - const s = Math.max(-1, Math.min(1, input[i])); - pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + this.buffer[this.length++] = input[i] * 0x7fff; + if (this.length === this.chunkSamples) { + this.port.postMessage(this.buffer, [this.buffer.buffer]); + this.buffer = new Int16Array(this.chunkSamples); + this.length = 0; + } } - this.port.postMessage(pcm16, [pcm16.buffer]); - return true; } } + registerProcessor("rawPcm16Processor", RawPcm16Processor); // #endregion pcm16-worklet diff --git a/frontend/src/pages/in-depth/dictate/dictate.ts b/frontend/src/pages/in-depth/dictate/dictate.ts index 625a34d..d39441c 100644 --- a/frontend/src/pages/in-depth/dictate/dictate.ts +++ b/frontend/src/pages/in-depth/dictate/dictate.ts @@ -36,13 +36,13 @@ import dictatePageSource from "./dictate.ts?raw"; // The "Code" tab is rendered from these real source regions — see #region markers. const CODE_SNIPPETS = [ { - title: "1. Capture mic audio in 100 ms PCM chunks", + title: "1. Wire up the microphone stream", file: "audio/mic-stream.ts", source: micStreamSource, region: "microphone-stream", }, { - title: "2. Convert mic audio to PCM-16 (AudioWorklet)", + title: "2. Convert and chunk mic audio in the AudioWorklet", file: "audio/rawPcm16Processor.js", source: rawPcm16ProcessorSource, region: "pcm16-worklet", diff --git a/frontend/src/pages/in-depth/transcribe/transcribe.ts b/frontend/src/pages/in-depth/transcribe/transcribe.ts index 1ce056a..628c788 100644 --- a/frontend/src/pages/in-depth/transcribe/transcribe.ts +++ b/frontend/src/pages/in-depth/transcribe/transcribe.ts @@ -35,13 +35,13 @@ import { // The "Code" tab is rendered from these real source regions — see #region markers. const CODE_SNIPPETS = [ { - title: "1. Capture mic audio in 100 ms PCM chunks", + title: "1. Wire up the microphone stream", file: "audio/mic-stream.ts", source: micStreamSource, region: "microphone-stream", }, { - title: "2. Convert mic audio to PCM-16 (AudioWorklet)", + title: "2. Convert and chunk mic audio in the AudioWorklet", file: "audio/rawPcm16Processor.js", source: rawPcm16ProcessorSource, region: "pcm16-worklet",