diff --git a/packages/core/src/schema.ts b/packages/core/src/schema.ts index a74d0bf..26477aa 100644 --- a/packages/core/src/schema.ts +++ b/packages/core/src/schema.ts @@ -14,6 +14,8 @@ const utteranceNode: NodeSpec = { startTimeSeconds: { default: null }, endTimeSeconds: { default: null }, continuationOfId: { default: null }, + intonation: { default: false }, // show a pitch contour above the words + intonationChannel: { default: null }, // per-block audio channel override (number | null) }, toDOM(node) { return ['p', { @@ -23,18 +25,23 @@ const utteranceNode: NodeSpec = { ...(node.attrs.tierId ? { 'data-tier-id': node.attrs.tierId } : {}), 'data-participant': node.attrs.participant, ...(node.attrs.continuationOfId ? { 'data-continuation-of': node.attrs.continuationOfId } : {}), + ...(node.attrs.intonation ? { 'data-intonation': 'true' } : {}), + ...(node.attrs.intonationChannel != null ? { 'data-intonation-channel': String(node.attrs.intonationChannel) } : {}), }, 0] }, parseDOM: [{ tag: 'p.utt', getAttrs(dom) { const el = dom + const chAttr = el.getAttribute('data-intonation-channel') return { id: el.getAttribute('data-id'), tier: el.getAttribute('data-tier') ?? '', tierId: el.getAttribute('data-tier-id') ?? null, participant: el.getAttribute('data-participant') ?? '', continuationOfId: el.getAttribute('data-continuation-of') ?? null, + intonation: el.getAttribute('data-intonation') === 'true', + intonationChannel: chAttr != null ? Number(chAttr) : null, } }, }], diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 8b845fc..48a8846 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -1178,6 +1178,16 @@ export class AnnotationStore extends TypedEmitter { this.ydoc.transact(() => { this.yParticipants.delete(id) }, USER_ORIGIN) } + /** Set (or clear, with null) a participant's default audio channel. */ + setParticipantChannel(id: ID, channel: number | null): void { + const p = this.yParticipants.get(id) + if (!p) return + const next = { ...p } + if (channel == null) delete next.channel + else next.channel = channel + this.ydoc.transact(() => { this.yParticipants.set(id, next) }, USER_ORIGIN) + } + getParticipant(id: ID): ParticipantJSON | undefined { return this.yParticipants.get(id) } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 9bc5020..f1f27b3 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -32,6 +32,8 @@ export interface ParticipantJSON { id: ID label: string attrs?: Record + /** Default audio channel index for this speaker (e.g. intonation contour source). */ + channel?: number } export interface TierDefJSON { diff --git a/packages/desktop/src/main/index.ts b/packages/desktop/src/main/index.ts index 93ae476..9a1bb1d 100644 --- a/packages/desktop/src/main/index.ts +++ b/packages/desktop/src/main/index.ts @@ -56,6 +56,10 @@ function createWindow(): BrowserWindow { if (!isMac) win.setMenuBarVisibility(false) if (process.env['ELECTRON_RENDERER_URL']) { + // [PERF] temporary: mirror renderer [PERF] logs to the terminal for easy copy/paste + win.webContents.on('console-message', (_e, _level, message) => { + if (message.startsWith('[PERF')) console.log(message) + }) void win.loadURL(process.env['ELECTRON_RENDERER_URL']) win.webContents.openDevTools() } else { diff --git a/packages/editor/src/TranscriptEditor.svelte b/packages/editor/src/TranscriptEditor.svelte index 1d93673..4e65639 100644 --- a/packages/editor/src/TranscriptEditor.svelte +++ b/packages/editor/src/TranscriptEditor.svelte @@ -43,6 +43,7 @@ import { buildAnchorPlugin, buildAnchorAlignmentPlugin } from './plugins/anchor.js' import { buildImageInputRulePlugin } from './plugins/image-command.js' import { buildSpectInputRulePlugin } from './commands/viz-commands.js' + import type { GetIntonation } from './nodeviews/ProsodyLayer.js' import { buildSymbolInputRulePlugin } from './plugins/symbol-input.js' import type { SymbolDef } from '@mumo/core' import type { VizContextMenuCallback } from './nodeviews/VisualizationNodeView.js' @@ -65,6 +66,9 @@ showEnd?: boolean onEscapeKey?: () => void getTokenTime?: (id: string) => { start: number; end: number } | undefined + getIntonation?: GetIntonation + getAudioChannels?: () => Array<{ index: number; label: string }> + getParticipantChannel?: (participant: string) => number | null editable?: boolean tokenClickMode?: boolean ontokenclick?: (token: TokenRecord) => void @@ -106,6 +110,9 @@ showEnd = false, onEscapeKey, getTokenTime, + getIntonation, + getAudioChannels, + getParticipantChannel, editable = true, tokenClickMode = false, ontokenclick, @@ -473,7 +480,7 @@ }, nodeViews: { utterance: (node, editorView, getPos) => - new UtteranceNodeView(node, editorView, getPos, onSeek), + new UtteranceNodeView(node, editorView, getPos, onSeek, tokenStore, getTokenTime, getIntonation, getAudioChannels, getParticipantChannel), visualization: (node, editorView, getPos) => new VisualizationNodeView(node, editorView, getPos, onSeek, onVizContextMenu), image: (node, editorView, getPos) => @@ -1380,6 +1387,76 @@ color: var(--color-primary, #4a90d9); } + :global(.utt-ctx-check) { + display: flex; + align-items: center; + gap: 0.5rem; + } + /* Checkbox indicator (boolean toggle, e.g. Intonation) */ + :global(.utt-ctx-box) { + width: 12px; + height: 12px; + border: 1.5px solid #aaa; + border-radius: 3px; + box-sizing: border-box; + flex-shrink: 0; + position: relative; + } + :global(.utt-ctx-box.on) { + background: var(--color-primary, #4a90d9); + border-color: var(--color-primary, #4a90d9); + } + :global(.utt-ctx-box.on::after) { + content: ''; + position: absolute; + left: 3px; + top: 0.5px; + width: 4px; + height: 7px; + border: solid #fff; + border-width: 0 1.5px 1.5px 0; + box-sizing: border-box; + transform: rotate(45deg); + } + /* Radio indicator (single choice, e.g. intonation channel) */ + :global(.utt-ctx-radio) { + width: 12px; + height: 12px; + border: 1.5px solid #aaa; + border-radius: 50%; + box-sizing: border-box; + flex-shrink: 0; + position: relative; + } + :global(.utt-ctx-radio.on) { + border-color: var(--color-primary, #4a90d9); + } + :global(.utt-ctx-radio.on::after) { + content: ''; + position: absolute; + left: 50%; + top: 50%; + width: 6px; + height: 6px; + border-radius: 50%; + background: var(--color-primary, #4a90d9); + transform: translate(-50%, -50%); + } + + :global(.utt-ctx-submenu-parent) { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.75rem; + } + + :global(.utt-ctx-arrow) { + color: var(--color-text-muted, #888); + font-size: 1rem; + line-height: 1; + margin-left: auto; + } + :global(.utt-tier) { flex-shrink: 0; width: var(--tier-col-w, 0rem); @@ -1415,6 +1492,36 @@ font-family: var(--transcript-font, 'CMU Serif', 'Computer Modern', Georgia, serif); } + /* Intonation contour: extra leading above each text line makes room for the band. */ + :global(.utt--intonation .utt-content) { + line-height: 3.1; + } + /* Reserve room above the FIRST line so its band doesn't bleed into the block above. + 24px = BAND_H (22) + BAND_GAP (2) in ProsodyLayer.ts. */ + :global(.utt--intonation) { + padding-top: 24px; + } + :global(.utt-intonation-overlay) { + position: absolute; + inset: 0; + pointer-events: none; + overflow: visible; + z-index: 1; + } + :global(.utt-intonation-svg) { + position: absolute; + top: 0; + left: 0; + overflow: visible; + } + :global(.utt-intonation-path) { + fill: none; + stroke: #2979ff; + stroke-width: 1.5; + stroke-linejoin: round; + stroke-linecap: round; + } + /* --utt-meta-w is defined on .transcript-editor using --ln-w */ :global(.utt-gloss) { diff --git a/packages/editor/src/index.ts b/packages/editor/src/index.ts index 07880b4..c2793ba 100644 --- a/packages/editor/src/index.ts +++ b/packages/editor/src/index.ts @@ -3,6 +3,7 @@ export type { FormattingState } from './format.js' export { setAllGlosses, setGlossesVisible, isGlossesVisible, getGlossEntryFor } from './gloss.js' export type { GlossEntry } from './gloss.js' export { setUttTiersVisible, isUttTiersVisible } from './utt-tier.js' +export { redrawAllProsody } from './prosody.js' export { default as TranscriptOverlay } from './TranscriptOverlay.svelte' export type { OverlayPlugin, OverlayContext } from './overlay.js' export { overlapAlignmentKey } from './plugins/overlap.js' @@ -13,6 +14,7 @@ export { UtteranceNodeView } from './nodeviews/UtteranceNodeView.js' export { ImageNodeView } from './nodeviews/ImageNodeView.js' export { VisualizationNodeView } from './nodeviews/VisualizationNodeView.js' export { buildImageInputRulePlugin } from './plugins/image-command.js' +export type { GetIntonation } from './nodeviews/ProsodyLayer.js' export { buildKeymapPlugin } from './commands/keymaps.js' export { buildSymbolInputRulePlugin, SYMBOL_COMMANDS, DEFAULT_SYMBOL_DEFS } from './plugins/symbol-input.js' export { runToggleBold, runToggleItalic, runToggleStrike, runToggleUnderline, runInsertChar, runInsertOverlapBracket, runSelectParentBlock, runApplyFont, runInsertComment } from './commands/toolbar-commands.js' diff --git a/packages/editor/src/nodeviews/ProsodyLayer.ts b/packages/editor/src/nodeviews/ProsodyLayer.ts new file mode 100644 index 0000000..9c12ee2 --- /dev/null +++ b/packages/editor/src/nodeviews/ProsodyLayer.ts @@ -0,0 +1,196 @@ +import type { Node } from 'prosemirror-model' +import type { EditorView } from 'prosemirror-view' +import type { TokenStore } from '@mumo/core' + +/** f0 samples for an utterance's time span, plus the y-scale to map Hz → band height. */ +export type GetIntonation = ( + channel: number, + t0: number, + t1: number, +) => { samples: Array<[number, number]>; yMin: number; yMax: number } | null + +export type GetTokenTime = (id: string) => { start: number; end: number } | undefined + +const SVG_NS = 'http://www.w3.org/2000/svg' +/** Height (px) of the contour band drawn above each visual text line. */ +const BAND_H = 22 +/** Gap (px) between the bottom of the band and the top of the text line. */ +const BAND_GAP = 2 + +/** A word measured on screen: its time window and pixel extent (overlay-local coords). */ +interface WordBox { + t0: number + t1: number + left: number + right: number + top: number + line: number +} + +/** + * Renders a pitch (f0) contour above the words of an utterance (Option 1 = "warp"): the contour is + * time-warped so each word's melodic shape sits above that word. Owned by UtteranceNodeView and + * only alive while `utterance.attrs.intonation` is true. Reads word times from the existing + * token-timing store; no bespoke storage. + */ +export class ProsodyLayer { + private overlay: HTMLDivElement + private svg: SVGSVGElement + + constructor( + private uttDom: HTMLElement, + private contentDOM: HTMLElement, + private view: EditorView, + private getPos: () => number | undefined, + private getNode: () => Node, + private tokenStore: TokenStore | undefined, + private getTokenTime: GetTokenTime | undefined, + private getIntonation: GetIntonation | undefined, + private getParticipantChannel: ((participant: string) => number | null) | undefined, + ) { + this.overlay = document.createElement('div') + this.overlay.className = 'utt-intonation-overlay' + this.overlay.contentEditable = 'false' + this.svg = document.createElementNS(SVG_NS, 'svg') + this.svg.setAttribute('class', 'utt-intonation-svg') + this.overlay.appendChild(this.svg) + this.uttDom.classList.add('utt--intonation') + this.uttDom.appendChild(this.overlay) + } + + /** + * True if the mutation/event originated inside the overlay (so the NodeView can ignore it). + * Must accept Node, not HTMLElement — the SVG contour elements are SVGElement, and missing them + * here makes ProseMirror re-render the node on every draw → infinite loop. + */ + contains(target: EventTarget | null): boolean { + // globalThis.Node: the bare `Node` name is the (type-only) prosemirror-model import in this file. + return target instanceof globalThis.Node && this.overlay.contains(target) + } + + destroy(): void { + this.overlay.remove() + this.uttDom.classList.remove('utt--intonation') + } + + /** Recompute and redraw the contour. Cheap enough to call on update/reflow. */ + draw(): void { + while (this.svg.firstChild) this.svg.removeChild(this.svg.firstChild) + + const node = this.getNode() + const uttStart = node.attrs.startTimeSeconds as number | null + const uttEnd = node.attrs.endTimeSeconds as number | null + if (uttStart == null || uttEnd == null || uttEnd <= uttStart) return + + // Resolve channel: per-block override, else the speaker's participant default, else 0. + const override = node.attrs.intonationChannel as number | null + const participant = (node.attrs.participant as string | null) ?? '' + const channel = override ?? this.getParticipantChannel?.(participant) ?? 0 + const contour = this.getIntonation?.(channel, uttStart, uttEnd) + if (!contour || contour.samples.length === 0) return + + const words = this._measureWords(node, uttStart, uttEnd) + if (words.length === 0) return + + const uttRect = this.uttDom.getBoundingClientRect() + this.svg.setAttribute('width', String(uttRect.width)) + this.svg.setAttribute('height', String(uttRect.height)) + this.svg.setAttribute('viewBox', `0 0 ${uttRect.width} ${uttRect.height}`) + + const { yMin, yMax } = contour + const span = yMax - yMin || 1 + + // Group words by visual line; draw one warped path per line so wrapping works. + const lines = new Map() + for (const w of words) { + const arr = lines.get(w.line) ?? [] + arr.push(w) + lines.set(w.line, arr) + } + + for (const lineWords of lines.values()) { + lineWords.sort((a, b) => a.left - b.left) + const bandBottom = Math.min(...lineWords.map(w => w.top)) - BAND_GAP + const bandTop = bandBottom - BAND_H + const lineT0 = Math.min(...lineWords.map(w => w.t0)) + const lineT1 = Math.max(...lineWords.map(w => w.t1)) + + let d = '' + let penDown = false + for (const [t, hz] of contour.samples) { + if (t < lineT0 || t > lineT1 || Number.isNaN(hz)) { penDown = false; continue } + const w = lineWords.find(w => t >= w.t0 && t <= w.t1) + if (!w) { penDown = false; continue } // in a gap between words + const frac = w.t1 > w.t0 ? (t - w.t0) / (w.t1 - w.t0) : 0 + const x = w.left + frac * (w.right - w.left) + const y = bandBottom - ((hz - yMin) / span) * BAND_H + const yc = Math.max(bandTop, Math.min(bandBottom, y)) + d += `${penDown ? 'L' : 'M'}${x.toFixed(1)} ${yc.toFixed(1)}` + penDown = true + } + if (!d) continue + const path = document.createElementNS(SVG_NS, 'path') + path.setAttribute('d', d) + path.setAttribute('class', 'utt-intonation-path') + this.svg.appendChild(path) + } + } + + /** Measure each word token's time window + on-screen pixel box (overlay-local coords). */ + private _measureWords(node: Node, uttStart: number, uttEnd: number): WordBox[] { + const pos = this.getPos() + if (pos === undefined) return [] + const contentStart = pos + 1 + const uttId = node.attrs.id as string + const tokens = this.tokenStore?.getUttTokens(uttId) ?? [] + const wordToks = tokens.filter(t => t.kind === 'word') + if (wordToks.length === 0) return [] + + const uttRect = this.uttDom.getBoundingClientRect() + const n = wordToks.length + const out: WordBox[] = [] + for (let i = 0; i < n; i++) { + const tok = wordToks[i]! + const time = this.getTokenTime?.(tok.id) + // Fall back to an even spread across the utterance (symbolic-subdivision semantics). + const t0 = time?.start ?? uttStart + ((uttEnd - uttStart) * i) / n + const t1 = time?.end ?? uttStart + ((uttEnd - uttStart) * (i + 1)) / n + + const startPos = this._posForOffset(node, contentStart, tok.startOffset) + const endPos = this._posForOffset(node, contentStart, tok.endOffset) + let a: { left: number; right: number; top: number; bottom: number } + let b: { left: number; right: number; top: number; bottom: number } + try { + a = this.view.coordsAtPos(startPos, 1) + b = this.view.coordsAtPos(endPos, -1) + } catch { continue } + + out.push({ + t0, t1, + left: a.left - uttRect.left, + right: b.right - uttRect.left, + top: a.top - uttRect.top, + line: Math.round(a.top), + }) + } + return out + } + + /** Map a character offset within the utterance's text to a PM document position. */ + private _posForOffset(node: Node, contentStart: number, targetChar: number): number { + let pos = contentStart + let chars = 0 + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i) + if (child.isText) { + const len = child.text?.length ?? 0 + if (chars + len >= targetChar) return pos + (targetChar - chars) + chars += len + pos += child.nodeSize + } else { + pos += child.nodeSize // atom (overlap bracket / image / inline_ann): 0 chars + } + } + return pos + } +} diff --git a/packages/editor/src/nodeviews/UtteranceNodeView.ts b/packages/editor/src/nodeviews/UtteranceNodeView.ts index d5f252b..f889610 100644 --- a/packages/editor/src/nodeviews/UtteranceNodeView.ts +++ b/packages/editor/src/nodeviews/UtteranceNodeView.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-argument */ import type { Node } from 'prosemirror-model' -import type { EditorView, NodeView } from 'prosemirror-view' +import type { EditorView, NodeView, ViewMutationRecord } from 'prosemirror-view' import { TextSelection } from 'prosemirror-state' import { formatTime, getCurrentDecimals, registerTimeView, unregisterTimeView } from '../format.js' import { startTimeEdit } from '../time-edit.js' @@ -8,6 +8,10 @@ import { startFieldEdit } from '../field-editor.js' import { registerGlossView, unregisterGlossView, getGlossEntryFor, isGlossesVisible } from '../gloss.js' import type { GlossEntry } from '../gloss.js' import { registerUttTierView, unregisterUttTierView, isUttTiersVisible } from '../utt-tier.js' +import { registerProsodyView, unregisterProsodyView } from '../prosody.js' +import { ProsodyLayer } from './ProsodyLayer.js' +import type { GetIntonation, GetTokenTime } from './ProsodyLayer.js' +import type { TokenStore } from '@mumo/core' // Module-level singleton so at most one context menu is open at a time. let _activeContextMenu: HTMLElement | null = null @@ -31,6 +35,12 @@ export class UtteranceNodeView implements NodeView { getPos: () => number | undefined private onSeek: ((t: number) => void) | undefined + private tokenStore: TokenStore | undefined + private getTokenTime: GetTokenTime | undefined + private getIntonation: GetIntonation | undefined + private getAudioChannels: (() => Array<{ index: number; label: string }>) | undefined + private getParticipantChannel: ((participant: string) => number | null) | undefined + private _prosody: ProsodyLayer | null = null _decimals: number _editingWhichTime: 'start' | 'end' | null = null @@ -42,11 +52,26 @@ export class UtteranceNodeView implements NodeView { private _glossEditing = false private _glossOriginalText = '' - constructor(node: Node, view: EditorView, getPos: () => number | undefined, onSeek?: ((t: number) => void) ) { + constructor( + node: Node, + view: EditorView, + getPos: () => number | undefined, + onSeek?: ((t: number) => void), + tokenStore?: TokenStore, + getTokenTime?: GetTokenTime, + getIntonation?: GetIntonation, + getAudioChannels?: () => Array<{ index: number; label: string }>, + getParticipantChannel?: (participant: string) => number | null, + ) { this.node = node this.view = view this.getPos = getPos this.onSeek = onSeek + this.tokenStore = tokenStore + this.getTokenTime = getTokenTime + this.getIntonation = getIntonation + this.getAudioChannels = getAudioChannels + this.getParticipantChannel = getParticipantChannel this._decimals = getCurrentDecimals() registerTimeView(this) @@ -161,6 +186,41 @@ export class UtteranceNodeView implements NodeView { this._cancelGlossEdit() } }) + + this._syncProsody() + registerProsodyView(this) + } + + // Intonation contour (prosody layer) + + /** Host hook (see prosody.ts): redraw the contour when the underlying pitch data changed. */ + redrawProsody(): void { + if (this._prosody) this._scheduleProsodyDraw() + } + + /** Create/destroy the contour layer to match the node's `intonation` attr, then redraw. */ + private _syncProsody(): void { + const on = this.node.attrs.intonation === true + if (on && !this._prosody) { + this._prosody = new ProsodyLayer( + this.dom, this.contentDOM, this.view, this.getPos, + () => this.node, this.tokenStore, this.getTokenTime, this.getIntonation, + this.getParticipantChannel, + ) + } else if (!on && this._prosody) { + this._prosody.destroy() + this._prosody = null + } + if (this._prosody) this._scheduleProsodyDraw() + } + + private _prosodyRaf = 0 + private _scheduleProsodyDraw(): void { + if (this._prosodyRaf) cancelAnimationFrame(this._prosodyRaf) + this._prosodyRaf = requestAnimationFrame(() => { + this._prosodyRaf = 0 + this._prosody?.draw() + }) } private _refreshSepEl(continuationOfId: string | null): void { @@ -372,6 +432,80 @@ export class UtteranceNodeView implements NodeView { }) menu.appendChild(newBtn) + const sep2 = document.createElement('div') + sep2.className = 'utt-ctx-sep' + menu.appendChild(sep2) + + const intonOn = this.node.attrs.intonation === true + const intonBtn = document.createElement('button') + intonBtn.className = 'utt-ctx-item utt-ctx-check' + intonBtn.innerHTML = `Intonation` + intonBtn.addEventListener('mousedown', (e) => { + e.preventDefault() + _closeActiveContextMenu() + this._setIntonation(!intonOn) + }) + menu.appendChild(intonBtn) + + // Intonation channel picker, as a hover submenu (only when there's a choice of channels). + const channels = this.getAudioChannels?.() ?? [] + if (channels.length > 1) { + const override = this.node.attrs.intonationChannel as number | null // null = use participant default + + const chItem = document.createElement('button') + chItem.className = 'utt-ctx-item utt-ctx-submenu-parent' + chItem.innerHTML = `Channel›` + + const submenu = document.createElement('div') + submenu.className = 'utt-ctx-menu utt-ctx-submenu' + submenu.style.cssText = 'position:fixed;z-index:10000;display:none' + + // Options: "Participant default" (clears the override) + one per channel. + const participant = (this.node.attrs.participant as string | null) ?? '' + const defCh = this.getParticipantChannel?.(participant) ?? null + const defLabel = defCh != null + ? (channels.find(c => c.index === defCh)?.label ?? `Ch ${defCh}`) + : 'none set' + const options: Array<{ value: number | null; label: string }> = [ + { value: null, label: `Participant default (${defLabel})` }, + ...channels.map(c => ({ value: c.index, label: c.label })), + ] + for (const { value, label } of options) { + const chBtn = document.createElement('button') + chBtn.className = 'utt-ctx-item utt-ctx-check' + const checked = value === override + chBtn.innerHTML = `${label}` + chBtn.addEventListener('mousedown', (e) => { + e.preventDefault() + _closeActiveContextMenu() + this._setIntonationChannel(value) + }) + submenu.appendChild(chBtn) + } + + let hideT = 0 + const showSub = () => { + clearTimeout(hideT) + const r = chItem.getBoundingClientRect() + submenu.style.display = '' + submenu.style.left = `${r.right - 2}px` + submenu.style.top = `${r.top}px` + requestAnimationFrame(() => { + const sr = submenu.getBoundingClientRect() + if (sr.right > window.innerWidth) submenu.style.left = `${r.left - sr.width + 2}px` + if (sr.bottom > window.innerHeight) submenu.style.top = `${Math.max(0, window.innerHeight - sr.height)}px` + }) + } + const hideSub = () => { hideT = window.setTimeout(() => { submenu.style.display = 'none' }, 140) } + chItem.addEventListener('mouseenter', showSub) + chItem.addEventListener('mouseleave', hideSub) + submenu.addEventListener('mouseenter', () => { clearTimeout(hideT) }) + submenu.addEventListener('mouseleave', hideSub) + + menu.appendChild(chItem) + menu.appendChild(submenu) // child of menu so the outside-click guard treats it as inside + } + document.body.appendChild(menu) _activeContextMenu = menu @@ -391,6 +525,20 @@ export class UtteranceNodeView implements NodeView { document.addEventListener('mousedown', onMousedown, true) } + private _setIntonation(on: boolean): void { + const pos = this.getPos() + if (pos === undefined) return + this.view.dispatch(this.view.state.tr.setNodeMarkup(pos, undefined, { ...this.node.attrs, intonation: on })) + } + + /** Pick the audio channel for this block's contour (null = participant default). Also turns it on. */ + private _setIntonationChannel(index: number | null): void { + const pos = this.getPos() + if (pos === undefined) return + this.view.dispatch(this.view.state.tr.setNodeMarkup(pos, undefined, + { ...this.node.attrs, intonation: true, intonationChannel: index })) + } + private _moveToParticipant(newParticipant: string): void { const pos = this.getPos() if (pos === undefined) return @@ -425,7 +573,8 @@ export class UtteranceNodeView implements NodeView { ) } - ignoreMutation(): boolean { + ignoreMutation(mutation: ViewMutationRecord): boolean { + if (this._prosody?.contains(mutation.target)) return true return ( this._participantEditing || this.tierEl.contentEditable === 'true' || @@ -446,6 +595,10 @@ export class UtteranceNodeView implements NodeView { unregisterTimeView(this) unregisterGlossView(this.node.attrs.id as string, this) unregisterUttTierView(this.node.attrs.id as string, this) + unregisterProsodyView(this) + if (this._prosodyRaf) cancelAnimationFrame(this._prosodyRaf) + this._prosody?.destroy() + this._prosody = null if (_activeContextMenu) _closeActiveContextMenu() } @@ -478,6 +631,7 @@ export class UtteranceNodeView implements NodeView { } else { this.dom.removeAttribute('data-continuation') } + this._syncProsody() return true } diff --git a/packages/editor/src/prosody.ts b/packages/editor/src/prosody.ts new file mode 100644 index 0000000..de1d04d --- /dev/null +++ b/packages/editor/src/prosody.ts @@ -0,0 +1,22 @@ +interface ProsodyView { + redrawProsody(): void +} + +const _registry = new Set() + +export function registerProsodyView(view: ProsodyView): void { + _registry.add(view) +} + +export function unregisterProsodyView(view: ProsodyView): void { + _registry.delete(view) +} + +/** + * Redraw every live intonation contour. Nodeviews read pitch data through the host's `getIntonation` + * callback but don't observe it, so the host must call this when that data changes out-of-band — + * late pitch compute, a loaded `.mumo` injecting persisted pitch, or a participant-channel change. + */ +export function redrawAllProsody(): void { + for (const view of _registry) view.redrawProsody() +} diff --git a/packages/media-player/assets/silero_vad_legacy.onnx b/packages/media-player/assets/silero_vad_legacy.onnx new file mode 100644 index 0000000..e6db48d Binary files /dev/null and b/packages/media-player/assets/silero_vad_legacy.onnx differ diff --git a/packages/media-player/assets/swift-f0.onnx b/packages/media-player/assets/swift-f0.onnx new file mode 100644 index 0000000..ba94e42 Binary files /dev/null and b/packages/media-player/assets/swift-f0.onnx differ diff --git a/packages/media-player/package.json b/packages/media-player/package.json index db57359..530e8f2 100644 --- a/packages/media-player/package.json +++ b/packages/media-player/package.json @@ -13,7 +13,7 @@ "dependencies": { "@mumo/core": "workspace:*", "@mumo/timeline": "workspace:*", - "@ricky0123/vad-web": "^0.0.30", + "onnxruntime-web": "^1.27.0", "mediabunny": "^1.50.3", "pixi.js": "^8.19.0", "audio-analysis-wasm": "workspace:*" diff --git a/packages/media-player/src/LinkedMediaDlg.svelte b/packages/media-player/src/LinkedMediaDlg.svelte index 1433666..c91738d 100644 --- a/packages/media-player/src/LinkedMediaDlg.svelte +++ b/packages/media-player/src/LinkedMediaDlg.svelte @@ -77,6 +77,7 @@ .lmd-header { display: flex; align-items: center; justify-content: space-between; padding: 10px 14px; border-bottom: 1px solid #eee; font-weight: 600; + cursor: move; user-select: none; } .lmd-close { background: none; border: none; cursor: pointer; opacity: 0.45; font-size: 14px; color: #222; diff --git a/packages/media-player/src/MediaPlayer.ts b/packages/media-player/src/MediaPlayer.ts index 1d31c5f..a772964 100644 --- a/packages/media-player/src/MediaPlayer.ts +++ b/packages/media-player/src/MediaPlayer.ts @@ -1,6 +1,6 @@ -import { Input, UrlSource, AudioBufferSink, ALL_FORMATS } from 'mediabunny' import type { SpectrogramTile, WaveformBins } from '@mumo/timeline' -import type { SpectrogramSettings, MediaState, MediaTrack, VadSegment, FrameStat } from './types.js' +import type { SpectrogramSettings, MediaState, MediaTrack, VadSegment, VadSettings, PitchTrack, PitchSettings, FrameStat } from './types.js' +import { DEFAULT_VAD_SETTINGS, DEFAULT_PITCH_SETTINGS } from './types.js' import type { PlatformIO } from './platform.js' import { SignalBroker } from './SignalBroker.js' import type { SignalCallbacks } from './SignalBroker.js' @@ -24,6 +24,9 @@ export interface MediaPlayerCallbacks { onSpectrogramTile(channelIndex: number, tile: SpectrogramTile): void onOnsets(channelIndex: number, timestamps: Float32Array, strengths: Float32Array, bandTimestamps: Float32Array[], bandStrengths: Float32Array[]): void onVad(segments: VadSegment[]): void + onVadProgress(done: number, total: number): void + onPitch(channelIndex: number, track: PitchTrack): void + onPitchProgress(done: number, total: number): void onProgress(done: number, total: number): void onError(message: string): void onCustom(pluginId: string, data: unknown): void @@ -39,7 +42,8 @@ export class MediaPlayer { private readonly _broker: SignalBroker private _paused = true private _clockFn: () => number = () => 0 - private _decodeId = 0 // cancels stale audio decode loops on reload + private _vadSettings: VadSettings = { ...DEFAULT_VAD_SETTINGS } + private _pitchSettings: PitchSettings = { ...DEFAULT_PITCH_SETTINGS } private readonly _stateListeners = new Set<(s: MediaState | null) => void>() private readonly _playingListeners = new Set<(playing: boolean) => void>() @@ -51,13 +55,18 @@ export class MediaPlayer { ) { const signalCallbacks: SignalCallbacks = { onDecoded: (sampleRate, channelCount, duration) => { - if (this.state) this._setState({ ...this.state, sampleRate, channelCount, duration }) + // For video, the renderer's duration is authoritative (audio & video tracks can differ + // in length); only fall back to the decoded audio duration for audio-only media. + if (this.state) this._setState({ ...this.state, sampleRate, channelCount, duration: this.state.duration || duration }) }, onWaveform: (ch, bins) => _callbacks.onWaveform?.(ch, bins), onSpectrogramOverview: (ch, tile) => _callbacks.onSpectrogramOverview?.(ch, tile), onSpectrogramTile: (ch, tile) => _callbacks.onSpectrogramTile?.(ch, tile), onOnsets: (ch, ts, str, bts, bstr) => _callbacks.onOnsets?.(ch, ts, str, bts, bstr), onVad: segs => _callbacks.onVad?.(segs), + onVadProgress: (d, t) => _callbacks.onVadProgress?.(d, t), + onPitch: (ch, track) => _callbacks.onPitch?.(ch, track), + onPitchProgress: (d, t) => _callbacks.onPitchProgress?.(d, t), onProgress: (d, t) => _callbacks.onProgress?.(d, t), onError: msg => _callbacks.onError?.(msg), onCustom: (id, data) => _callbacks.onCustom?.(id, data), @@ -141,9 +150,8 @@ export class MediaPlayer { if (this.track?.file === file) this.track = { ...this.track, mediaHash: hash } }) - // Start worker for this file; then stream decoded audio chunks to it - this._broker.startStream(settings) - void this._streamAudio(mediaUrl, ++this._decodeId) + // Decode + analyze the whole file in the worker (off the main thread). + this._broker.analyze(mediaUrl, settings, { __vadSettings: this._vadSettings, __pitch: this._pitchSettings }) if (kind === 'audio') { // Audio-only files never get a visible canvas tile, so create an offscreen renderer @@ -177,8 +185,7 @@ export class MediaPlayer { this.track = this.track?.mediaUrl === url ? this.track : null this._setState({ mediaUrl: url, kind, filename: name, duration: 0, sampleRate: 0, channelCount: 1, activeChannel: 'mix', muted: this.state?.muted ?? false, volume: this.state?.volume ?? 1 }) - this._broker.startStream(settings) - void this._streamAudio(url, ++this._decodeId) + this._broker.analyze(url, settings, { __vadSettings: this._vadSettings, __pitch: this._pitchSettings }) if (kind === 'audio') { if (!this._videoRenderer) { @@ -200,49 +207,25 @@ export class MediaPlayer { } } - /** - * Decode audio from a URL on the main thread (AudioBuffer requires window context) - * and stream each chunk to the worker without accumulating PCM in main memory. - */ - private async _streamAudio(url: string, decodeId: number): Promise { - const name = this.state?.filename ?? url.split('/').pop() ?? 'media' - const source = new UrlSource(url) - const input = new Input({ formats: ALL_FORMATS, source }) - try { - const at = await input.getPrimaryAudioTrack() - if (!at || decodeId !== this._decodeId) return - const sampleRate = await at.getSampleRate() - if (decodeId !== this._decodeId) return - console.log(`[media] audio processing start: ${name} (${sampleRate} Hz)`) - const sink = new AudioBufferSink(at) - let duration = 0, channelCount = 0 - for await (const { buffer, timestamp } of sink.buffers(0)) { - if (decodeId !== this._decodeId) break - channelCount = buffer.numberOfChannels - duration = Math.max(duration, timestamp + buffer.duration) - // Extract and transfer each channel — zero-copy, no accumulation on main thread - const chunks: Float32Array[] = [] - for (let ch = 0; ch < channelCount; ch++) { - chunks.push(buffer.getChannelData(ch).slice()) - } - this._broker.feedChunk(chunks, sampleRate, channelCount) - } - if (decodeId === this._decodeId) { - console.log(`[media] audio processing done: ${name} (${duration.toFixed(2)}s, ${channelCount}ch)`) - this._broker.endStream(duration) - } - } catch (err) { - if (decodeId === this._decodeId) { - console.error(`[media] audio processing error: ${name}`, err) - this._callbacks.onError?.(`Failed to decode audio: ${String(err)}`) - } - } finally { - input.dispose() - } - } - reanalyze(settings: SpectrogramSettings): void { this._broker.reanalyze(settings) } + /** Remember the current VAD settings so the next analyze segments with them. */ + setVadSettings(settings: VadSettings): void { this._vadSettings = { ...settings } } + + /** Apply new VAD settings now: re-segment the worker's cached probs (instant, no re-decode). */ + resegmentVad(settings: VadSettings): void { this._vadSettings = { ...settings }; this._broker.resegmentVad(this._vadSettings) } + + /** Compute VAD now as a deferred pass (re-decode, VAD-only). Used lazily when VAD is enabled. */ + computeVad(): void { this._broker.analyzeVad(this._vadSettings) } + + // Copy to a plain object: callers may pass a framework reactive proxy (e.g. Svelte $state), which + // is not structured-cloneable and would break postMessage to the worker. + /** Remember the current pitch settings so the next analyze uses them. */ + setPitchSettings(settings: PitchSettings): void { this._pitchSettings = { ...settings } } + + /** Re-run pitch detection with new settings (re-decodes; backend/frequency/threshold changes). */ + reanalyzePitch(settings: PitchSettings): void { this._pitchSettings = { ...settings }; this._broker.reanalyzePitch(this._pitchSettings) } + captureFrame(): Promise { return this._videoRenderer?.captureFrame() ?? Promise.resolve(null) } diff --git a/packages/media-player/src/MultiMediaPlayer.ts b/packages/media-player/src/MultiMediaPlayer.ts index 948e27e..9d575f0 100644 --- a/packages/media-player/src/MultiMediaPlayer.ts +++ b/packages/media-player/src/MultiMediaPlayer.ts @@ -1,6 +1,6 @@ import type { SpectrogramTile, WaveformBins } from '@mumo/timeline' -import type { SpectrogramSettings, MediaState, VadSegment } from './types.js' -import { DEFAULT_SPEC_SETTINGS } from './types.js' +import type { SpectrogramSettings, MediaState, VadSegment, VadSettings, PitchTrack, PitchSettings } from './types.js' +import { DEFAULT_VAD_SETTINGS, DEFAULT_SPEC_SETTINGS, DEFAULT_PITCH_SETTINGS } from './types.js' import type { PlatformIO } from './platform.js' import { MediaPlayer } from './MediaPlayer.js' import type { MediaPlayerCallbacks } from './MediaPlayer.js' @@ -17,6 +17,9 @@ export interface MultiMediaPlayerCallbacks { onSpectrogramTile(playerId: string, channelIndex: number, tile: SpectrogramTile): void onOnsets(playerId: string, channelIndex: number, timestamps: Float32Array, strengths: Float32Array, bandTimestamps: Float32Array[], bandStrengths: Float32Array[]): void onVad(segments: VadSegment[]): void + onVadProgress(done: number, total: number): void + onPitch(playerId: string, channelIndex: number, track: PitchTrack): void + onPitchProgress(done: number, total: number): void onProgress(done: number, total: number): void onError(message: string): void onCustom(pluginId: string, data: unknown): void @@ -27,6 +30,8 @@ export interface MultiMediaPlayerCallbacks { export class MultiMediaPlayer { private _players: MediaPlayer[] = [] private _settings: SpectrogramSettings = { ...DEFAULT_SPEC_SETTINGS } + private _vadSettings: VadSettings = { ...DEFAULT_VAD_SETTINGS } + private _pitchSettings: PitchSettings = { ...DEFAULT_PITCH_SETTINGS } private readonly _playersListeners = new Set<(players: readonly MediaPlayer[]) => void>() private _lastPrimaryId: string | null = null @@ -345,6 +350,29 @@ export class MultiMediaPlayer { for (const p of this._players) p.reanalyze(settings) } + /** Apply VAD segmentation settings to all players — re-segments cached probs instantly. */ + setVadSettings(settings: VadSettings): void { + this._vadSettings = settings + for (const p of this._players) p.resegmentVad(settings) + } + + /** Remember pitch settings on all players for the next compute — does NOT re-run detection. */ + setPitchSettings(settings: PitchSettings): void { + this._pitchSettings = settings + for (const p of this._players) p.setPitchSettings(settings) + } + + /** Compute pitch now for all players (re-decode, pitch-only). Used lazily when the overlay is + * turned on or when detection settings change while it's shown. */ + computePitch(): void { + for (const p of this._players) p.reanalyzePitch(this._pitchSettings) + } + + /** Compute VAD now for all players (deferred re-decode, VAD-only). */ + computeVad(): void { + for (const p of this._players) p.computeVad() + } + dispose(): void { cancelAnimationFrame(this._rafId) for (const p of this._players) p.dispose() @@ -437,12 +465,17 @@ export class MultiMediaPlayer { onSpectrogramTile: (ch, tile) => this._callbacks.onSpectrogramTile?.(player.id, ch, tile), onOnsets: (ch, ts, str, bts, bstr) => this._callbacks.onOnsets?.(player.id, ch, ts, str, bts, bstr), onVad: segs => { if (player === this._players[0]) this._callbacks.onVad?.(segs) }, + onVadProgress: (d, t) => { if (player === this._players[0]) this._callbacks.onVadProgress?.(d, t) }, + onPitch: (ch, track) => this._callbacks.onPitch?.(player.id, ch, track), + onPitchProgress: (d, t) => { if (player === this._players[0]) this._callbacks.onPitchProgress?.(d, t) }, onProgress: (d, t) => { if (player === this._players[0]) this._callbacks.onProgress?.(d, t) }, onError: msg => this._callbacks.onError?.(msg), onCustom: (id, data) => { if (player === this._players[0]) this._callbacks.onCustom?.(id, data) }, } player = new MediaPlayer(cbs, this._platform, this._workerUrl) player._setClockFn(() => this.getPlaybackTime()) + player.setVadSettings(this._vadSettings) + player.setPitchSettings(this._pitchSettings) return player } } diff --git a/packages/media-player/src/SignalBroker.ts b/packages/media-player/src/SignalBroker.ts index e8b2005..d96be9d 100644 --- a/packages/media-player/src/SignalBroker.ts +++ b/packages/media-player/src/SignalBroker.ts @@ -1,5 +1,5 @@ import type { SpectrogramTile, WaveformBins } from '@mumo/timeline' -import type { SpectrogramSettings, WorkerResponse, VadSegment } from './types.js' +import type { SpectrogramSettings, WorkerResponse, VadSegment, VadSettings, PitchTrack, PitchSettings } from './types.js' export interface SignalCallbacks { onDecoded(sampleRate: number, channelCount: number, duration: number): void @@ -8,6 +8,9 @@ export interface SignalCallbacks { onSpectrogramTile(channelIndex: number, tile: SpectrogramTile): void onOnsets(channelIndex: number, timestamps: Float32Array, strengths: Float32Array, bandTimestamps: Float32Array[], bandStrengths: Float32Array[]): void onVad(segments: VadSegment[]): void + onVadProgress(done: number, total: number): void + onPitch(channelIndex: number, track: PitchTrack): void + onPitchProgress(done: number, total: number): void onProgress(done: number, total: number): void onError(message: string): void onCustom(pluginId: string, data: unknown): void @@ -15,9 +18,9 @@ export interface SignalCallbacks { /** * Manages the signal analysis worker. - * Audio decoding happens on the main thread (AudioBuffer requires the window context); - * decoded chunks are streamed to the worker via feedChunk/endStream so the main thread - * never accumulates the full PCM buffer. + * The worker decodes the media file itself (via mediabunny over the `media://` protocol) + * and analyzes it — all off the main thread, so a multi-minute whole-file scan never + * competes with playback or UI rendering. */ export class SignalBroker { private _worker: Worker | null = null @@ -27,25 +30,24 @@ export class SignalBroker { private readonly _workerUrl?: string, ) {} - /** Start a new analysis for a new file. Terminates any previous worker. */ - startStream(settings: SpectrogramSettings, pluginSettings?: Record): void { + /** Start decoding + analyzing a new file in the worker. Terminates any previous worker. */ + analyze(url: string, settings: SpectrogramSettings, pluginSettings?: Record): void { this._worker?.terminate() this._worker = this._createWorker() - this._worker.postMessage({ type: 'initStream', settings, pluginSettings }) + // Resolve VAD asset URLs here (renderer): document.baseURI points at the app root — next to + // which the public model + ort wasm are served — in dev, web, and packaged Electron alike. + // The worker can't resolve these reliably itself (its import.meta.url is under assets/). + this._worker.postMessage({ type: 'analyze', url, settings, pluginSettings: { ...pluginSettings, ...this._assets() } }) } - /** Send one decoded audio chunk. Buffers are transferred (zero-copy). */ - feedChunk(channelData: Float32Array[], sampleRate: number, channelCount: number): void { - if (!this._worker) return - this._worker.postMessage( - { type: 'chunk', channelData, sampleRate, channelCount }, - channelData.map(c => c.buffer), - ) - } - - /** Signal that all chunks have been sent and analysis should begin. */ - endStream(duration: number): void { - this._worker?.postMessage({ type: 'finalizeStream', duration }) + /** ONNX asset URLs (Silero + SwiftF0 models, ort wasm dir), resolved against document.baseURI — + * the app root next to which the public assets are served in dev, web, and packaged Electron. */ + private _assets(): Record { + const wasmBase = new URL('./', document.baseURI).href + return { + __vad: { modelUrl: new URL('silero_vad_legacy.onnx', document.baseURI).href, wasmBase }, + __pitchAssets: { modelUrl: new URL('swift-f0.onnx', document.baseURI).href, wasmBase }, + } } /** Re-run spectrogram analysis with new settings using the worker's stored audio. */ @@ -53,6 +55,21 @@ export class SignalBroker { this._worker?.postMessage({ type: 'reanalyze', settings }) } + /** Re-derive VAD segments from the worker's cached per-frame probs with new settings (instant). */ + resegmentVad(vadSettings: VadSettings): void { + this._worker?.postMessage({ type: 'resegmentVad', vadSettings }) + } + + /** Re-run pitch detection with new settings (re-decodes; backend/frequency/threshold changes). */ + reanalyzePitch(pitchSettings: PitchSettings): void { + this._worker?.postMessage({ type: 'reanalyzePitch', pitchSettings, pluginSettings: this._assets() }) + } + + /** Compute VAD as a deferred pass (re-decodes) so it doesn't delay spectrogram/waveform. */ + analyzeVad(vadSettings: VadSettings): void { + this._worker?.postMessage({ type: 'analyzeVad', vadSettings, pluginSettings: this._assets() }) + } + dispose(): void { this._worker?.terminate() this._worker = null @@ -83,6 +100,12 @@ export class SignalBroker { this._callbacks.onOnsets(msg.channelIndex, msg.timestamps, msg.strengths, msg.bandTimestamps, msg.bandStrengths); break case 'vad': this._callbacks.onVad(msg.segments); break + case 'vadProgress': + this._callbacks.onVadProgress(msg.done, msg.total); break + case 'pitch': + this._callbacks.onPitch(msg.channelIndex, msg.track); break + case 'pitchProgress': + this._callbacks.onPitchProgress(msg.done, msg.total); break case 'progress': this._callbacks.onProgress(msg.done, msg.total); break case 'error': diff --git a/packages/media-player/src/VideoRenderer.ts b/packages/media-player/src/VideoRenderer.ts index 6564242..4840ee0 100644 --- a/packages/media-player/src/VideoRenderer.ts +++ b/packages/media-player/src/VideoRenderer.ts @@ -347,31 +347,35 @@ export class VideoRenderer { const iter = this._videoSink.canvases(localTime) this._videoIter = iter - const firstResult = await iter.next() - if (id !== this._asyncId) return - if (!firstResult.done) this._applyFrame(firstResult.value) - - // Pre-buffer so tick() has frames immediately when play() starts - const minPreBuffered = Math.min(3, FRAME_QUEUE_SIZE - 1) - while (this._frameQueue.length < minPreBuffered) { - const r = await iter.next() + try { + const firstResult = await iter.next() if (id !== this._asyncId) return - if (r.done) break - this._frameQueue.push(r.value) - } + if (!firstResult.done) this._applyFrame(firstResult.value) + + // Pre-buffer so tick() has frames immediately when play() starts + const minPreBuffered = Math.min(3, FRAME_QUEUE_SIZE - 1) + while (this._frameQueue.length < minPreBuffered) { + const r = await iter.next() + if (id !== this._asyncId) return + if (r.done) break + this._frameQueue.push(r.value) + } + } catch { return /* input disposed/reloaded mid-prebuffer — normal teardown */ } void this._fillQueue(iter, id) } private async _fillQueue(iter: AsyncGenerator, id: number): Promise { - for await (const frame of iter) { - if (id !== this._asyncId) break - this._frameQueue.push(frame) - if (this._frameQueue.length >= FRAME_QUEUE_SIZE) { - await new Promise(r => { this._queueSpace = r }) + try { + for await (const frame of iter) { if (id !== this._asyncId) break + this._frameQueue.push(frame) + if (this._frameQueue.length >= FRAME_QUEUE_SIZE) { + await new Promise(r => { this._queueSpace = r }) + if (id !== this._asyncId) break + } } - } + } catch { /* input disposed while streaming frames — normal teardown */ } } /** Consume queued frames whose file-local time ≤ (globalTimeSec - offset). Returns true if frame was drawn. */ diff --git a/packages/media-player/src/index.ts b/packages/media-player/src/index.ts index 7083c59..a19bfe9 100644 --- a/packages/media-player/src/index.ts +++ b/packages/media-player/src/index.ts @@ -1,10 +1,10 @@ -export type { SpectrogramSettings, MediaState, MediaTrack, WorkerRequest, WorkerResponse, VadSegment, SpectrogramTile, WaveformBins, FrameStat } from './types.js' -export { SPEC_PRESETS, DEFAULT_SPEC_SETTINGS, PREVIEW_SPEC_SETTINGS } from './types.js' +export type { SpectrogramSettings, MediaState, MediaTrack, WorkerRequest, WorkerResponse, VadSegment, VadSettings, PitchTrack, PitchSettings, PitchBackend, SpectrogramTile, WaveformBins, FrameStat } from './types.js' +export { SPEC_PRESETS, DEFAULT_SPEC_SETTINGS, PREVIEW_SPEC_SETTINGS, DEFAULT_VAD_SETTINGS, DEFAULT_PITCH_SETTINGS } from './types.js' export type { PlatformIO, DesktopPlatformIO, FontEntry, SystemFonts } from './platform.js' export { guessMime, isDesktop } from './platform.js' -export type { SignalPlugin, AudioCtx, SignalPost } from './plugins/signal/SignalPlugin.js' +export type { SignalPlugin, SignalRun, StreamInit, AudioSegment, SignalPost } from './plugins/signal/SignalPlugin.js' export type { VideoPlugin } from './plugins/video/VideoPlugin.js' export { MediaPlayer } from './MediaPlayer.js' @@ -30,3 +30,4 @@ export * as PIXI from 'pixi.js' export { TrackOverlayPlugin, TRACK_COLORS } from './plugins/video/TrackOverlayPlugin.js' export type { VizOptions } from './plugins/video/TrackOverlayPlugin.js' export { computeEnergyVad } from './plugins/signal/vad.js' +export { smoothOctaves } from './plugins/signal/praatPitch.js' diff --git a/packages/media-player/src/mediaWorker.ts b/packages/media-player/src/mediaWorker.ts index 2a4dafe..a940889 100644 --- a/packages/media-player/src/mediaWorker.ts +++ b/packages/media-player/src/mediaWorker.ts @@ -1,9 +1,12 @@ /// +import { Input, UrlSource, AudioSampleSink, ALL_FORMATS } from 'mediabunny' import type { WorkerRequest, WorkerResponse, SpectrogramSettings } from './types.js' -import type { SignalPlugin, AudioCtx, SignalPost } from './plugins/signal/SignalPlugin.js' -import { setSampleBuffer } from './plugins/signal/spectrogram.js' +import type { SignalPlugin, SignalRun, AudioSegment, SignalPost } from './plugins/signal/SignalPlugin.js' +import { setSampleBuffer, spectrogramPlugin, specFrameParams } from './plugins/signal/spectrogram.js' import { waveformPlugin } from './plugins/signal/waveform.js' -import { spectrogramPlugin } from './plugins/signal/spectrogram.js' +import { sileroVadPlugin, resegmentVad } from './plugins/signal/sileroVad.js' +import { pitchPlugin } from './plugins/signal/pitch.js' +import { SegmentProducer } from './segmenter.js' async function loadWasm(): Promise { try { @@ -18,19 +21,8 @@ async function loadWasm(): Promise { } } -function mixChannels(channels: Float32Array[], settings: SpectrogramSettings): Float32Array[] { - if (!settings.monoMix || channels.length <= 1) return channels - const len = channels[0]!.length - const mixed = new Float32Array(len) - const n = channels.length - for (let i = 0; i < len; i++) { - let sum = 0 - for (const ch of channels) sum += ch[i]! - mixed[i] = sum / n - } - return [mixed] -} - +// VAD (Silero/ONNX) and pitch each run as their own deferred decode pass, so the fast +// spectrogram/waveform appear first instead of being blocked by per-segment model inference. const defaultPlugins: SignalPlugin[] = [waveformPlugin, spectrogramPlugin] const _wasmPromise = loadWasm() @@ -38,20 +30,81 @@ const post: SignalPost = (msg: WorkerResponse, transfer?: Transferable[]) => { self.postMessage(msg, transfer ?? []) } -// Accumulated state across streaming messages -let _chunks: Float32Array[][] = [] // per-channel chunk lists -let _sampleRate = 0 -let _channelCount = 0 -let _settings: SpectrogramSettings | null = null +// Retained for reanalyze / reanalyzePitch (re-decode; no PCM is kept). +let _url: string | null = null let _pluginSettings: Record = {} +let _settings: SpectrogramSettings | null = null // last spectrogram settings, reused for segment framing -// Stored after analysis for reanalyze -let _channels: Float32Array[] | null = null -let _duration = 0 +const SEGMENT_SEC = 90 // ~1.5 min per segment + +async function decodeAndAnalyze( + url: string, settings: SpectrogramSettings, pluginSettings: Record, + trigger: 'analyze' | 'reanalyze', plugins: SignalPlugin[], postDecoded = true, +): Promise { + const input = new Input({ formats: ALL_FORMATS, source: new UrlSource(url) }) + try { + const at = await input.getPrimaryAudioTrack() + // A video with no audio track is normal, not an error — just skip audio analysis; the video + // still plays and its duration comes from the video renderer. + if (!at) { console.log('[worker] no audio track — skipping audio analysis'); return } + const durationSec = await at.computeDuration() + const sink = new AudioSampleSink(at) + const gen = sink.samples() + const first = await gen.next() + if (first.done) { console.log('[worker] no audio samples — skipping audio analysis'); return } + + const srcChannels = first.value.numberOfChannels + const sampleRate = first.value.sampleRate + const mono = settings.monoMix && srcChannels > 1 + const channelCount = mono ? 1 : srcChannels + // The pitch second pass (and pitch re-run) skip this: re-posting 'decoded' would re-fire the + // primary state-change handler, which clears the first pass's spectrogram/waveform signals. + if (postDecoded) post({ type: 'decoded', sampleRate, channelCount, duration: durationSec }) + + const { hop, windowSize, tileFrames } = specFrameParams(settings, sampleRate) + const totalSamples = Math.max(windowSize, Math.round(durationSec * sampleRate)) + const totalFrames = Math.max(1, Math.floor((totalSamples - windowSize) / hop) + 1) + const framesPerSeg = Math.round(SEGMENT_SEC * sampleRate / hop) + const segFrames = Math.max(tileFrames, Math.round(framesPerSeg / tileFrames) * tileFrames) + + const runs: SignalRun[] = plugins.map(p => p.createRun({ sampleRate, channelCount, durationSec, settings, pluginSettings, trigger }, post)) + const producer = new SegmentProducer(channelCount, hop, windowSize, segFrames, totalFrames) + + const pushSample = (s: import('mediabunny').AudioSample): void => { + const frames = s.numberOfFrames + const perCh: Float32Array[] = [] + if (mono) { + const acc = new Float32Array(frames) + const tmp = new Float32Array(frames) + for (let ch = 0; ch < srcChannels; ch++) { s.copyTo(tmp, { planeIndex: ch, format: 'f32-planar' }); for (let i = 0; i < frames; i++) acc[i]! += tmp[i]! } + for (let i = 0; i < frames; i++) acc[i]! /= srcChannels + perCh.push(acc) + } else { + for (let ch = 0; ch < channelCount; ch++) { const arr = new Float32Array(frames); s.copyTo(arr, { planeIndex: ch, format: 'f32-planar' }); perCh.push(arr) } + } + producer.add(perCh, frames) + s.close() + } + + const drain = async (final: boolean): Promise => { + let seg: AudioSegment | null + while ((seg = producer.tryCut(final)) !== null) for (const r of runs) await r.pushSegment(seg) + } + + pushSample(first.value) + await drain(false) + for await (const s of gen) { pushSample(s); await drain(false) } + await drain(true) + for (const r of runs) await r.finish() + } catch (err) { + post({ type: 'error', message: String(err) }) + } finally { + input.dispose() + } +} -// Serialize message processing: chain each handler onto the previous so they -// never interleave at await points (prevents double-analysis when reanalyze -// arrives while finalizeStream's VAD await is in progress). +// Serialize message processing: chain each handler onto the previous so a reanalyze that +// arrives mid-analyze runs after it, not concurrently. let _queue: Promise = Promise.resolve() self.onmessage = (e: MessageEvent) => { @@ -61,64 +114,27 @@ self.onmessage = (e: MessageEvent) => { async function handleMessage(req: WorkerRequest): Promise { await _wasmPromise - if (req.type === 'initStream') { - _chunks = [] - _sampleRate = 0 - _channelCount = 0 - _settings = req.settings + if (req.type === 'analyze') { + _url = req.url _pluginSettings = req.pluginSettings ?? {} - - } else if (req.type === 'chunk') { - _sampleRate = req.sampleRate - _channelCount = req.channelCount - for (let ch = 0; ch < req.channelData.length; ch++) { - if (!_chunks[ch]) _chunks[ch] = [] - _chunks[ch]!.push(req.channelData[ch]!) - } - - } else if (req.type === 'finalizeStream') { - if (!_settings) return - try { - _duration = req.duration - - // Concatenate per-channel chunks into flat arrays - _channels = _chunks.map(chunks => { - const total = chunks.reduce((n, c) => n + c.length, 0) - const out = new Float32Array(total) - let pos = 0 - for (const c of chunks) { out.set(c, pos); pos += c.length } - return out - }) - _chunks = [] // free chunk references - - post({ type: 'decoded', sampleRate: _sampleRate, channelCount: _channelCount, duration: _duration }) - - const ctx: AudioCtx = { - channels: mixChannels(_channels, _settings), sampleRate: _sampleRate, duration: _duration, - channelCount: _channelCount, settings: _settings, pluginSettings: _pluginSettings, - trigger: 'analyze', - } - for (const plugin of defaultPlugins) { - await plugin.analyze(ctx, post) - } - } catch (err) { - post({ type: 'error', message: String(err) }) - } - - } else { // req.type === 'reanalyze' - // Always update stored settings so that if reanalyze arrives before - // finalizeStream, the initial analysis uses the latest settings. _settings = req.settings - if (!_channels) return - try { - const ctx: AudioCtx = { - channels: mixChannels(_channels, req.settings), sampleRate: _sampleRate, duration: _duration, - channelCount: _channelCount, settings: req.settings, pluginSettings: _pluginSettings, - trigger: 'reanalyze', - } - await spectrogramPlugin.analyze(ctx, post) - } catch (err) { - post({ type: 'error', message: String(err) }) - } + await decodeAndAnalyze(req.url, req.settings, _pluginSettings, 'analyze', defaultPlugins) + // Pitch is computed lazily (on demand) via reanalyzePitch — NOT on load. + } else if (req.type === 'resegmentVad') { + // Re-threshold the cached per-frame probs — no decode or model inference. + resegmentVad(req.vadSettings, post) + } else if (req.type === 'analyzeVad') { + // Deferred VAD pass: re-decode and run only Silero VAD (waveform/spectrogram unchanged). + if (!_url || !_settings) return + const ps = { ..._pluginSettings, ...(req.pluginSettings ?? {}), __vadSettings: req.vadSettings } + await decodeAndAnalyze(_url, _settings, ps, 'analyze', [sileroVadPlugin], false) + } else if (req.type === 'reanalyzePitch') { + // Re-decode and re-run only pitch with new settings (waveform/spectrogram/VAD unchanged). + if (!_url || !_settings) return + const ps = { ..._pluginSettings, ...(req.pluginSettings ?? {}), __pitch: req.pitchSettings } + await decodeAndAnalyze(_url, _settings, ps, 'analyze', [pitchPlugin], false) + } else { // reanalyze — re-decode with new spectrogram settings (waveform/VAD are settings-independent) + if (!_url) return + await decodeAndAnalyze(_url, req.settings, _pluginSettings, 'reanalyze', [spectrogramPlugin]) } } diff --git a/packages/media-player/src/plugins/signal/SignalPlugin.ts b/packages/media-player/src/plugins/signal/SignalPlugin.ts index f586c01..53b2cc2 100644 --- a/packages/media-player/src/plugins/signal/SignalPlugin.ts +++ b/packages/media-player/src/plugins/signal/SignalPlugin.ts @@ -1,18 +1,44 @@ import type { WorkerResponse, SpectrogramSettings } from '../../types.js' -export interface AudioCtx { - channels: Float32Array[] +export type SignalPost = (msg: WorkerResponse, transfer?: Transferable[]) => void + +/** Per-run context. No PCM here — audio arrives segment by segment via {@link SignalRun.pushSegment}. */ +export interface StreamInit { sampleRate: number - duration: number channelCount: number + /** Total media duration in seconds (from container metadata), known up front. */ + durationSec: number settings: SpectrogramSettings pluginSettings: Record trigger: 'analyze' | 'reanalyze' } -export type SignalPost = (msg: WorkerResponse, transfer?: Transferable[]) => void +/** + * A contiguous window of decoded PCM produced by the worker. + * + * `channels[c]` covers absolute sample range `[startSample, startSample + channels[c].length)`. + * Segments are aligned to the spectrogram frame/tile grid and carry a trailing window overlap so + * the last owned frame is complete — that overlap is re-sent at the start of the next segment. + * `ownedSamples` is the count of leading, non-overlapping samples this segment owns; consumers that + * don't care about frames (waveform, VAD downmix) must only read `[0, ownedSamples)` to avoid + * double-counting the overlap. + */ +export interface AudioSegment { + channels: Float32Array[] + startSample: number // absolute index of channels[c][0] (== firstFrame * hop) + ownedSamples: number // leading non-overlapping samples owned by this segment + firstFrame: number // global spectrogram frame index of local frame 0 + frameCount: number // complete frames owned by this segment + isLast: boolean +} + +/** A stateful analysis run: fed ordered segments, then finalized. */ +export interface SignalRun { + pushSegment(seg: AudioSegment): void | Promise + finish(): void | Promise +} export interface SignalPlugin { id: string - analyze(ctx: AudioCtx, post: SignalPost): Promise + createRun(init: StreamInit, post: SignalPost): SignalRun } diff --git a/packages/media-player/src/plugins/signal/pitch.ts b/packages/media-player/src/plugins/signal/pitch.ts new file mode 100644 index 0000000..0ffefc9 --- /dev/null +++ b/packages/media-player/src/plugins/signal/pitch.ts @@ -0,0 +1,237 @@ +import type { PitchSettings, PitchTrack } from '../../types.js' +import { DEFAULT_PITCH_SETTINGS } from '../../types.js' +import type { SignalPlugin, SignalRun, StreamInit, AudioSegment, SignalPost } from './SignalPlugin.js' +import { getSampleBuffer } from './spectrogram.js' + +// Rust YIN (audio-analysis-wasm SampleBuffer.compute_pitch) runs at 16 kHz. FRAME=1024 (64 ms) +// supports down to ~50 Hz (max lag 320 < FRAME/2); HOP=256 → 16 ms frame spacing. +const PITCH_SR = 16000 +const FRAME = 1024 +const HOP = 256 + +function readPitchSettings(pluginSettings: Record): PitchSettings { + return { ...DEFAULT_PITCH_SETTINGS, ...((pluginSettings as { __pitch?: Partial }).__pitch ?? {}) } +} + +/** + * Streaming box-filter resampler to 16 kHz producing a *continuous* stream (not fixed frames): + * each `push` returns all output samples it can complete, carrying leftover input across calls so + * the concatenation of pushes is a seamless resampling. Bounded memory. (The frame-emitting + * `Resampler16k` in sileroVad is shaped for the VAD model; pitch needs a continuous stream.) + */ +export class ContinuousResampler16k { + private inBuf = new Float32Array(0) + private inBase = 0 // global native index of inBuf[0] + private outIndex = 0 // next global 16 kHz output index to produce + private readonly ratio: number // native samples per output sample + + constructor(nativeRate: number) { this.ratio = nativeRate / PITCH_SR } + + push(input: Float32Array): Float32Array { + if (input.length > 0) { + const buf = new Float32Array(this.inBuf.length + input.length) + buf.set(this.inBuf); buf.set(input, this.inBuf.length) + this.inBuf = buf + } + const end = this.inBase + this.inBuf.length + const out: number[] = [] + // Complete every output sample whose native window [lo, hi) is fully buffered. + let hi = Math.round((this.outIndex + 1) * this.ratio) + while (hi <= end) { + const lo = Math.round(this.outIndex * this.ratio) + let sum = 0, num = 0 + for (let i = lo; i < hi; i++) { sum += this.inBuf[i - this.inBase]!; num++ } + out.push(num > 0 ? sum / num : 0) + this.outIndex++ + hi = Math.round((this.outIndex + 1) * this.ratio) + } + // Drop native consumed by completed outputs, keeping the tail the next output needs. + const keepFrom = Math.round(this.outIndex * this.ratio) + if (keepFrom > this.inBase) { this.inBuf = this.inBuf.subarray(keepFrom - this.inBase); this.inBase = keepFrom } + return Float32Array.from(out) + } +} + +// SwiftF0: 16 kHz mono in, internal STFT (hop 256 → 16 ms frames), frame center offset 127.5. +// Processed in non-overlapping 30 s chunks (a multiple of the hop) to bound memory. +const SWIFT_HOP = 256 +const SWIFT_PAD = 127.5 +const SWIFT_CHUNK = 30 * PITCH_SR +const SWIFT_MIN = FRAME // don't run the model on a tail shorter than one STFT window + +type Ort = typeof import('onnxruntime-web') + +interface VadAssetsLike { modelUrl: string; wasmBase: string } +function readPitchAssets(pluginSettings: Record): VadAssetsLike | undefined { + return (pluginSettings as { __pitchAssets?: VadAssetsLike }).__pitchAssets +} + +function appendF32(a: Float32Array, b: Float32Array): Float32Array { + if (b.length === 0) return a + const out = new Float32Array(a.length + b.length) + out.set(a); out.set(b, a.length) + return out +} + +interface ChannelPitch { + resampler: ContinuousResampler16k + carry: Float32Array + start16k: number // global 16 kHz sample index of carry[0] (SwiftF0 frame-time base) + f0: number[] + confidence: number[] + times: number[] +} + +function makeChannels(init: StreamInit): ChannelPitch[] { + return Array.from({ length: init.channelCount }, () => ({ + resampler: new ContinuousResampler16k(init.sampleRate), carry: new Float32Array(0), start16k: 0, f0: [], confidence: [], times: [], + })) +} + +function postTrack(post: SignalPost, ch: number, backend: PitchTrack['backend'], c: ChannelPitch): void { + const f0 = Float32Array.from(c.f0), confidence = Float32Array.from(c.confidence), times = Float32Array.from(c.times) + post({ type: 'pitch', channelIndex: ch, track: { channelIndex: ch, backend, times, f0, confidence } }, [times.buffer, f0.buffer, confidence.buffer]) +} + +// Progress by owned samples processed vs the file's estimated total. `done()` posts 100% so the +// bar clears even when pitch is skipped (WASM/model unavailable). +function makeProgress(init: StreamInit, post: SignalPost) { + const total = Math.max(1, Math.round(init.durationSec * init.sampleRate)) + let processed = 0 + return { + tick(ownedSamples: number): void { processed += ownedSamples; post({ type: 'pitchProgress', done: Math.min(processed, total), total }) }, + done(): void { post({ type: 'pitchProgress', done: total, total }) }, + } +} + +// Rust YIN — frame-local, synchronous. Continuous 16 kHz stream fed into compute_pitch as complete +// frames accumulate. Uniform frame grid (frame f centered at FRAME/2 + f*HOP). +function createYinRun(init: StreamInit, post: SignalPost, s: PitchSettings): SignalRun { + const SB = getSampleBuffer() + const nc = init.channelCount + let disabled = SB === null + if (SB === null) console.warn('[pitch] Rust/WASM not loaded — skipping YIN pitch') + const chans = disabled ? [] : makeChannels(init) + const progress = makeProgress(init, post) + + const drain = (c: ChannelPitch): void => { + if (c.carry.length < FRAME) return + const sb = new SB!(c.carry, FRAME, 0) + try { + const pr = sb.compute_pitch(HOP, FRAME, PITCH_SR, s.minHz, s.maxHz, s.threshold) + try { + const n = pr.num_frames + if (n > 0) { + const ph = pr.take_pitch_hz(), cf = pr.take_confidence() + for (let i = 0; i < n; i++) { c.f0.push(ph[i]!); c.confidence.push(cf[i]!) } + c.carry = c.carry.subarray(n * HOP) + } + } finally { pr.free() } + } finally { sb.free() } + } + + return { + pushSegment(seg: AudioSegment): void { + if (disabled) return + try { + for (let ch = 0; ch < nc; ch++) { + const c = chans[ch]! + c.carry = appendF32(c.carry, c.resampler.push(seg.channels[ch]!.subarray(0, seg.ownedSamples))) + drain(c) + } + } catch (e) { disabled = true; console.warn('[pitch] YIN error, disabling:', e) } + progress.tick(seg.ownedSamples) + }, + finish(): void { + progress.done() + if (disabled) return + const hopSec = HOP / PITCH_SR, t0Sec = (FRAME / 2) / PITCH_SR + for (let ch = 0; ch < nc; ch++) { + const c = chans[ch]! + for (let f = 0; f < c.f0.length; f++) c.times.push(t0Sec + f * hopSec) + postTrack(post, ch, 'yin', c) + } + }, + } +} + +// SwiftF0 — ONNX (onnxruntime-web, shared with Silero). Stateless STFT, so 30 s chunks are run +// independently; each frame's absolute time comes from the chunk's global 16 kHz offset. +function createSwiftRun(init: StreamInit, post: SignalPost, assets: VadAssetsLike | undefined): SignalRun { + const nc = init.channelCount + let disabled = !assets + if (!assets) console.warn('[pitch] SwiftF0 assets missing — skipping') + const chans = disabled ? [] : makeChannels(init) + const progress = makeProgress(init, post) + let ort: Ort | null = null + let session: import('onnxruntime-web').InferenceSession | null = null + let inName = '', outPitch = '', outConf = '' + + const ready = (async () => { + if (disabled) return + try { + ort = await import('onnxruntime-web') + ort.env.wasm.numThreads = 1 + ort.env.wasm.wasmPaths = assets!.wasmBase + const bytes = await (await fetch(assets!.modelUrl)).arrayBuffer() + session = await ort.InferenceSession.create(bytes, { executionProviders: ['wasm'], graphOptimizationLevel: 'all' }) + inName = session.inputNames[0]!; outPitch = session.outputNames[0]!; outConf = session.outputNames[1]! + } catch (e) { disabled = true; console.warn('[pitch] SwiftF0 disabled, skipping:', e) } + })() + + const runChunk = async (c: ChannelPitch, chunk: Float32Array, start16k: number): Promise => { + const t = new ort!.Tensor('float32', chunk, [1, chunk.length]) + const out = await session!.run({ [inName]: t }) + const pd = out[outPitch]!.data as Float32Array, cd = out[outConf]!.data as Float32Array + for (let i = 0; i < pd.length; i++) { + c.f0.push(pd[i]!); c.confidence.push(cd[i]!) + c.times.push((start16k + i * SWIFT_HOP + SWIFT_PAD) / PITCH_SR) + } + } + + return { + async pushSegment(seg: AudioSegment): Promise { + await ready + if (disabled) return + try { + for (let ch = 0; ch < nc; ch++) { + const c = chans[ch]! + c.carry = appendF32(c.carry, c.resampler.push(seg.channels[ch]!.subarray(0, seg.ownedSamples))) + while (c.carry.length >= SWIFT_CHUNK) { + await runChunk(c, c.carry.subarray(0, SWIFT_CHUNK), c.start16k) + c.carry = c.carry.subarray(SWIFT_CHUNK); c.start16k += SWIFT_CHUNK + } + } + } catch (e) { disabled = true; console.warn('[pitch] SwiftF0 inference error, disabling:', e) } + progress.tick(seg.ownedSamples) + }, + async finish(): Promise { + await ready + progress.done() + if (disabled) return + for (let ch = 0; ch < nc; ch++) { + const c = chans[ch]! + if (c.carry.length >= SWIFT_MIN) { try { await runChunk(c, c.carry, c.start16k) } catch { /* drop tail */ } } + postTrack(post, ch, 'swiftf0', c) + } + }, + } +} + +/** + * Per-channel pitch detection in the streaming pipeline (RustYIN default, SwiftF0 optional). Each + * segment's owned samples are resampled to a continuous 16 kHz stream and analyzed as they arrive, + * so no full-resolution (or full 16 kHz) audio is retained. The derived track is transient — posted + * at finish, re-derived on load, never persisted. + */ +export const pitchPlugin: SignalPlugin = { + id: 'pitch', + + createRun(init: StreamInit, post: SignalPost): SignalRun { + if (init.trigger === 'reanalyze') return { pushSegment() { /* pitch not part of spectrogram reanalyze */ }, finish() {} } + const settings = readPitchSettings(init.pluginSettings) + return settings.backend === 'swiftf0' + ? createSwiftRun(init, post, readPitchAssets(init.pluginSettings)) + : createYinRun(init, post, settings) + }, +} diff --git a/packages/media-player/src/plugins/signal/praatPitch.ts b/packages/media-player/src/plugins/signal/praatPitch.ts new file mode 100644 index 0000000..00a3e30 --- /dev/null +++ b/packages/media-player/src/plugins/signal/praatPitch.ts @@ -0,0 +1,288 @@ +// A faithful port of Praat's autocorrelation pitch method (Boersma 1993) + Viterbi path finder, +// from fon/Sound_to_Pitch.cpp and fon/Pitch.cpp. This is the "gold standard" pitch: per-frame AC +// normalized by the window's own autocorrelation, candidate maxima, then a cross-frame Viterbi path +// (octave-jump / voiced-unvoiced costs) that removes octave errors and stabilizes voicing. +// +// Simplification vs Praat: candidate frequency/strength use parabolic interpolation of the AC peak +// rather than Praat's sinc interpolation (NUM_interpolate_sinc). This is close enough to track Praat +// and to drive the same path decisions; sinc refinement can be added later for exact strengths. + +export interface PraatPitchParams { + timeStep: number // s; 0 = auto (periodsPerWindow / floor / 4) + pitchFloor: number // Hz + pitchCeiling: number // Hz + maxCandidates: number + silenceThreshold: number + voicingThreshold: number + octaveCost: number + octaveJumpCost: number + voicedUnvoicedCost: number +} + +export const PRAAT_PITCH_DEFAULTS: PraatPitchParams = { + timeStep: 0, pitchFloor: 75, pitchCeiling: 600, maxCandidates: 15, + silenceThreshold: 0.03, voicingThreshold: 0.45, octaveCost: 0.01, octaveJumpCost: 0.35, voicedUnvoicedCost: 0.14, +} + +export interface PraatPitchResult { + times: Float32Array + f0: Float32Array // Hz; 0 = unvoiced + strength: Float32Array // chosen candidate strength ([0,1]-ish) +} + +const log2 = (x: number): number => Math.log(x) / Math.LN2 + +/** + * Octave-jump correction for single-f0 detectors (e.g. SwiftF0, which has no candidates for a full + * Viterbi path). Snaps each voiced frame toward the local median octave, removing isolated octave + * errors while leaving real intonation untouched. A light analogue of Praat's path finder. + * Returns a corrected copy; 0 (unvoiced) is preserved. + */ +export function smoothOctaves(f0: Float32Array, halfWindow = 3): Float32Array { + const n = f0.length + const out = Float32Array.from(f0) + const win: number[] = [] + for (let i = 0; i < n; i++) { + if (f0[i]! <= 0) continue + win.length = 0 + for (let j = Math.max(0, i - halfWindow); j <= Math.min(n - 1, i + halfWindow); j++) if (f0[j]! > 0) win.push(f0[j]!) + if (win.length === 0) continue + win.sort((a, b) => a - b) + const med = win[win.length >> 1]! + let f = f0[i]! + while (f > med * Math.SQRT2) f /= 2 // > half-octave above median ⇒ octave-up error + while (f < med / Math.SQRT2) f *= 2 // > half-octave below median ⇒ octave-down error + out[i] = f + } + return out +} + +// ---- radix-2 complex FFT (in place). dir = -1 forward, +1 inverse (unnormalized). ---- +function fft(re: Float64Array, im: Float64Array, dir: number): void { + const n = re.length + for (let i = 1, j = 0; i < n; i++) { + let bit = n >> 1 + for (; j & bit; bit >>= 1) j ^= bit + j ^= bit + if (i < j) { const tr = re[i]!; re[i] = re[j]!; re[j] = tr; const ti = im[i]!; im[i] = im[j]!; im[j] = ti } + } + for (let len = 2; len <= n; len <<= 1) { + const ang = dir * 2 * Math.PI / len + const wr = Math.cos(ang), wi = Math.sin(ang) + for (let i = 0; i < n; i += len) { + let cr = 1, ci = 0 + for (let k = 0; k < len / 2; k++) { + const a = i + k, b = i + k + len / 2 + const xr = re[b]! * cr - im[b]! * ci, xi = re[b]! * ci + im[b]! * cr + re[b] = re[a]! - xr; im[b] = im[a]! - xi + re[a] = re[a]! + xr; im[a] = im[a]! + xi + const ncr = cr * wr - ci * wi; ci = cr * wi + ci * wr; cr = ncr + } + } + } +} + +// Real autocorrelation of `sig` (length nfft, zero-padded), via FFT: IFFT(|FFT|²). Scale is +// arbitrary (callers normalize by lag 0), so FFT normalization is irrelevant. +function autocorrelation(sig: Float64Array, nfft: number): Float64Array { + const re = new Float64Array(nfft), im = new Float64Array(nfft) + re.set(sig) + fft(re, im, -1) + for (let k = 0; k < nfft; k++) { re[k] = re[k]! * re[k]! + im[k]! * im[k]!; im[k] = 0 } + fft(re, im, 1) + return re // re[lag] = autocorrelation at lag (up to scale) +} + +export function praatPitchAc(samples: Float32Array | Float64Array, sampleRate: number, params: PraatPitchParams): PraatPitchResult { + const p = params + const dx = 1 / sampleRate + const nx = samples.length + const periodsPerWindow = 3.0 // AC (Hanning) + const dt = p.timeStep > 0 ? p.timeStep : periodsPerWindow / p.pitchFloor / 4 + const ceiling = Math.min(p.pitchCeiling, 0.5 / dx) + const maxnCandidates = Math.max(p.maxCandidates, Math.floor(ceiling / p.pitchFloor)) + + const nsamp_period = Math.floor(1 / dx / p.pitchFloor) + const dt_window = periodsPerWindow / p.pitchFloor + let nsamp_window = Math.floor(dt_window / dx) + const halfnsamp_window = Math.floor(nsamp_window / 2) - 1 + nsamp_window = halfnsamp_window * 2 + const halfnsamp_period = Math.floor(nsamp_period / 2) + 1 + + const maximumLag = Math.min(Math.floor(nsamp_window / periodsPerWindow) + 2, nsamp_window) + const interpolationDepth = 0.5 + let nsampFFT = 1 + while (nsampFFT < nsamp_window * (1 + interpolationDepth)) nsampFFT *= 2 + const brent_ixmax = Math.floor(nsamp_window * interpolationDepth) + + // Global peak (for silence/voicing). + let mean = 0 + for (let i = 0; i < nx; i++) mean += samples[i]! + mean /= nx || 1 + let globalPeak = 0 + for (let i = 0; i < nx; i++) { const v = Math.abs(samples[i]! - mean); if (v > globalPeak) globalPeak = v } + + // Hanning window (Praat form), and its normalized autocorrelation. + const window = new Float64Array(nsamp_window + 1) // 1-indexed + for (let i = 1; i <= nsamp_window; i++) window[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (nsamp_window + 1)) + const windowPad = new Float64Array(nsampFFT) + for (let i = 1; i <= nsamp_window; i++) windowPad[i - 1] = window[i]! + const windowAc = autocorrelation(windowPad, nsampFFT) + const windowR = new Float64Array(brent_ixmax + 2) + windowR[0] = 1 + for (let i = 1; i <= brent_ixmax + 1; i++) windowR[i] = windowAc[i]! / windowAc[0]! + + // Frame timing (Sampled_shortTermAnalysis): fit windows symmetrically in the duration. + const duration = nx * dx + const numberOfFrames = Math.max(1, Math.floor((duration - dt_window) / dt) + 1) + const t1 = 0.5 * duration - 0.5 * numberOfFrames * dt + 0.5 * dt // centre of frame 0 (x1 = 0.5 dx) + + interface Cand { frequency: number; strength: number } + const frames: { intensity: number; candidates: Cand[] }[] = [] + + const frameBuf = new Float64Array(nsampFFT) + for (let iframe = 0; iframe < numberOfFrames; iframe++) { + const t = t1 + iframe * dt + const leftSample = Math.floor(t / dx + 0.5) // 1-indexed; Sampled_xToLowIndex with x1 = 0.5 dx + const rightSample = leftSample + 1 + + // Local mean over ±nsamp_period. + let lmStart = rightSample - nsamp_period, lmEnd = leftSample + nsamp_period + if (lmStart < 1) lmStart = 1 + if (lmEnd > nx) lmEnd = nx + let localMean = 0 + for (let i = lmStart; i <= lmEnd; i++) localMean += samples[i - 1]! + localMean /= 2 * nsamp_period + + // Windowed frame (minus local mean), zero-padded. + const startSample = rightSample - halfnsamp_window + frameBuf.fill(0) + let localPeak = 0 + const peakStart = Math.max(1, halfnsamp_window + 1 - halfnsamp_period) + const peakEnd = Math.min(nsamp_window, halfnsamp_window + halfnsamp_period) + for (let j = 1; j <= nsamp_window; j++) { + const si = startSample + j - 1 // 1-indexed source sample + const s = si >= 1 && si <= nx ? samples[si - 1]! : 0 + const v = (s - localMean) * window[j]! + frameBuf[j - 1] = v + if (j >= peakStart && j <= peakEnd) { const a = Math.abs(v); if (a > localPeak) localPeak = a } + } + const intensity = localPeak > globalPeak ? 1 : localPeak / (globalPeak || 1) + + const candidates: Cand[] = [{ frequency: 0, strength: 0 }] // unvoiced always present + + if (localPeak > 0) { + const ac = autocorrelation(frameBuf, nsampFFT) + const r = new Float64Array(brent_ixmax + 2) + r[0] = 1 + for (let i = 1; i <= brent_ixmax + 1; i++) r[i] = ac[i]! / (ac[0]! * windowR[i]!) + + const iMax = Math.min(maximumLag, brent_ixmax) + for (let i = 2; i < iMax; i++) { + if (r[i]! > 0.5 * p.voicingThreshold && r[i]! > r[i - 1]! && r[i]! >= r[i + 1]!) { + const dr = 0.5 * (r[i + 1]! - r[i - 1]!) + const d2r = (r[i]! - r[i - 1]!) + (r[i]! - r[i + 1]!) + if (d2r <= 0) continue + const frequency = 1 / dx / (i + dr / d2r) + if (frequency <= 0) continue + let strength = r[i]! + 0.5 * dr * dr / d2r // parabolic peak value (≈ Praat's sinc strength) + if (strength > 1) strength = 1 / strength + + // Place candidate (grow to maxnCandidates, else replace the weakest by octave-adjusted strength). + let place = -1 + if (candidates.length < maxnCandidates) { + candidates.push({ frequency, strength }) + place = candidates.length - 1 + } else { + let weakest = 2 + for (let iw = 1; iw < maxnCandidates; iw++) { + const ls = candidates[iw]!.strength - p.octaveCost * log2(p.pitchFloor / candidates[iw]!.frequency) + if (ls < weakest) { weakest = ls; place = iw } + } + if (strength - p.octaveCost * log2(p.pitchFloor / frequency) <= weakest) place = -1 + if (place >= 0) candidates[place] = { frequency, strength } + } + } + } + } + frames.push({ intensity, candidates }) + } + + pathFinder(frames, dt, ceiling, p) + + // Extract the chosen path (candidate 0 after path finding is the winner; see pathFinder). + const times = new Float32Array(numberOfFrames) + const f0 = new Float32Array(numberOfFrames) + const strength = new Float32Array(numberOfFrames) + for (let i = 0; i < numberOfFrames; i++) { + times[i] = t1 + i * dt + const c = frames[i]!.candidates[0]! + f0[i] = c.frequency + strength[i] = c.strength + } + return { times, f0, strength } +} + +// Viterbi path finder (Pitch_pathFinder). Chooses one candidate per frame maximizing +// Σ delta − Σ transitionCost, then moves the winner to candidates[0]. +function pathFinder( + frames: { intensity: number; candidates: { frequency: number; strength: number }[] }[], + dt: number, ceiling: number, p: PraatPitchParams, +): void { + const n = frames.length + if (n === 0) return + const timeStepCorrection = 0.01 / dt + const octaveJumpCost = p.octaveJumpCost * timeStepCorrection + const voicedUnvoicedCost = p.voicedUnvoicedCost * timeStepCorrection + const voiced = (f: number): boolean => f > 0 && f < ceiling + + const delta: number[][] = [] + const psi: number[][] = [] + for (let iframe = 0; iframe < n; iframe++) { + const frame = frames[iframe]! + let unvoiced = p.silenceThreshold <= 0 ? 0 : 2 - frame.intensity / (p.silenceThreshold / (1 + p.voicingThreshold)) + unvoiced = p.voicingThreshold + Math.max(0, unvoiced) + const d = new Array(frame.candidates.length) + for (let c = 0; c < frame.candidates.length; c++) { + const cand = frame.candidates[c]! + d[c] = voiced(cand.frequency) ? cand.strength - p.octaveCost * log2(ceiling / cand.frequency) : unvoiced + } + delta.push(d) + psi.push(new Array(frame.candidates.length).fill(0)) + } + + for (let iframe = 1; iframe < n; iframe++) { + const prev = frames[iframe - 1]!, cur = frames[iframe]! + const prevDelta = delta[iframe - 1]!, curDelta = delta[iframe]!, curPsi = psi[iframe]! + for (let c2 = 0; c2 < cur.candidates.length; c2++) { + const f2 = cur.candidates[c2]!.frequency + let best = -1e30, place = 0 + for (let c1 = 0; c1 < prev.candidates.length; c1++) { + const f1 = prev.candidates[c1]!.frequency + const pv = !voiced(f1), cv = !voiced(f2) + let transition: number + if (cv) transition = pv ? 0 : voicedUnvoicedCost + else transition = pv ? voicedUnvoicedCost : octaveJumpCost * Math.abs(log2(f1 / f2)) + const value = prevDelta[c1]! - transition + curDelta[c2]! + if (value > best) { best = value; place = c1 } + } + curDelta[c2] = best + curPsi[c2] = place + } + } + + // Backtrack: record the chosen candidate index per frame (no mutation during backtrack). + const chosen = new Array(n) + let place = 0, best = delta[n - 1]![0]! + for (let c = 1; c < delta[n - 1]!.length; c++) if (delta[n - 1]![c]! > best) { best = delta[n - 1]![c]!; place = c } + for (let iframe = n - 1; iframe >= 0; iframe--) { + chosen[iframe] = place + if (iframe > 0) place = psi[iframe]![place]! + } + // Move each frame's winner to slot 0 so the caller reads candidates[0]. + for (let iframe = 0; iframe < n; iframe++) { + const cands = frames[iframe]!.candidates + const c = chosen[iframe]! + if (c !== 0) { const tmp = cands[0]!; cands[0] = cands[c]!; cands[c] = tmp } + } +} diff --git a/packages/media-player/src/plugins/signal/sileroVad.ts b/packages/media-player/src/plugins/signal/sileroVad.ts new file mode 100644 index 0000000..b2ae937 --- /dev/null +++ b/packages/media-player/src/plugins/signal/sileroVad.ts @@ -0,0 +1,192 @@ +import type { VadSegment, VadSettings } from '../../types.js' +import { DEFAULT_VAD_SETTINGS } from '../../types.js' +import type { SignalPlugin, SignalRun, StreamInit, AudioSegment, SignalPost } from './SignalPlugin.js' +import { mergeVadSegments } from './vad.js' + +// The Silero legacy model runs on 1536-sample frames at 16 kHz (96 ms per frame). A 16 kHz sample +// count divided by 16 gives milliseconds (16000 samples/s ÷ 1000 = 16 samples/ms). +const TARGET_RATE = 16000 +const FRAME_SAMPLES = 1536 +const MS_PER_FRAME = FRAME_SAMPLES / 16 // 96 + +/** URLs for the ONNX model + ort wasm, resolved on the main thread (see SignalBroker) where + * document.baseURI reliably points at the app root in dev, web, and packaged Electron. */ +export interface VadAssets { modelUrl: string; wasmBase: string } + +function readVadSettings(pluginSettings: Record): VadSettings { + return { ...DEFAULT_VAD_SETTINGS, ...((pluginSettings as { __vadSettings?: Partial }).__vadSettings ?? {}) } +} + +/** + * Streaming resampler to 16 kHz that emits complete `frameSize`-sample frames. Each output sample + * is the average of the native samples in its window — the same box-filter downsampling vad-web + * feeds Silero (a crude anti-alias; plain interpolation would alias and degrade the model). Input + * is fed segment-by-segment; only a fraction of a frame of leftover is buffered, so memory stays + * O(frameSize) — the full-resolution audio is never resident. + */ +export class Resampler16k { + private inBuf = new Float32Array(0) + + constructor(private readonly nativeRate: number, private readonly frameSize = FRAME_SAMPLES) {} + + private hasEnoughForFrame(): boolean { + return (this.inBuf.length * TARGET_RATE) / this.nativeRate >= this.frameSize + } + + private generateFrame(): Float32Array { + const out = new Float32Array(this.frameSize) + let inputIndex = 0 + for (let o = 0; o < this.frameSize; o++) { + let sum = 0, num = 0 + const bound = Math.min(this.inBuf.length, ((o + 1) * this.nativeRate) / TARGET_RATE) + while (inputIndex < bound) { sum += this.inBuf[inputIndex]!; num++; inputIndex++ } + out[o] = num > 0 ? sum / num : 0 + } + this.inBuf = this.inBuf.slice(inputIndex) + return out + } + + push(input: Float32Array): Float32Array[] { + if (input.length === 0) return [] + const buf = new Float32Array(this.inBuf.length + input.length) + buf.set(this.inBuf); buf.set(input, this.inBuf.length) + this.inBuf = buf + + const frames: Float32Array[] = [] + while (this.hasEnoughForFrame()) frames.push(this.generateFrame()) + return frames + } +} + +/** + * Hysteresis segmentation over one channel's per-frame speech probabilities — a compact port of + * vad-web's FrameProcessor (minus the audio-buffer payload we don't need). A segment opens once a + * frame exceeds `positiveThreshold` and closes after `redemptionMs` of sub-`negativeThreshold` + * frames; segments shorter than `minSpeechMs` of speech are discarded. Operating on the cached + * probs means re-running this with new settings is instant — no decode or model inference. + */ +export function segmentProbs(probs: Float32Array | number[], s: VadSettings): VadSegment[] { + const redemptionFrames = Math.max(1, Math.round(s.redemptionMs / MS_PER_FRAME)) + const minSpeechFrames = Math.max(1, Math.round(s.minSpeechMs / MS_PER_FRAME)) + const toSec = (frame: number): number => (frame * MS_PER_FRAME) / 1000 + const segments: VadSegment[] = [] + let speaking = false, redemption = 0, speechFrames = 0, startFrame = 0, lastSpeechFrame = 0 + for (let f = 0; f < probs.length; f++) { + const prob = probs[f]! + const isSpeech = prob >= s.positiveThreshold + if (isSpeech) { speechFrames++; redemption = 0; lastSpeechFrame = f } + if (isSpeech && !speaking) { speaking = true; startFrame = f } + if (prob < s.negativeThreshold && speaking && ++redemption >= redemptionFrames) { + redemption = 0 + speaking = false + // End at the last frame that actually contained speech, not the redemption-expiry frame — + // the ~redemptionMs grace only decides *whether* to end, it shouldn't pad the reported end. + if (speechFrames >= minSpeechFrames) segments.push({ start: toSec(startFrame), end: toSec(lastSpeechFrame + 1) }) + speechFrames = 0 + } + if (!speaking) speechFrames = 0 + } + if (speaking && speechFrames >= minSpeechFrames) segments.push({ start: toSec(startFrame), end: toSec(lastSpeechFrame + 1) }) + return segments +} + +// Per-channel speech probabilities from the most recent analyze, retained so a settings change can +// re-segment without re-decoding. Cleared when a new file is analyzed (the worker is recreated). +let _probCache: number[][] | null = null + +/** Re-derive VAD segments from the cached probs with new settings and post them. No-op (leaves the + * existing VAD in place) if nothing has been analyzed yet. */ +export function resegmentVad(settings: VadSettings, post: SignalPost): void { + if (!_probCache) return + post({ type: 'vad', segments: mergeVadSegments(_probCache.map(probs => segmentProbs(probs, settings))) }) +} + +type Ort = typeof import('onnxruntime-web') +type Tensor = import('onnxruntime-web').Tensor +interface ChannelVad { + resampler: Resampler16k + probs: number[] + h: Tensor + c: Tensor +} + +/** + * Per-channel Silero VAD, run in the streaming pipeline. Each decoded segment's owned samples are + * resampled to 16 kHz and fed frame-by-frame to the Silero model (one shared, stateless ONNX + * session; the recurrent h/c state is carried per channel), so no full-resolution audio is + * retained. Per-channel speech segments are merged at finish. If the model or ort runtime can't + * load, the run disables itself and posts nothing — the energy VAD emitted by the spectrogram + * plugin then stands. + */ +export const sileroVadPlugin: SignalPlugin = { + id: 'silero-vad', + + createRun(init: StreamInit, post: SignalPost): SignalRun { + const assets = (init.pluginSettings as { __vad?: VadAssets }).__vad + const settings = readVadSettings(init.pluginSettings) + const nc = init.channelCount + let disabled = init.trigger === 'reanalyze' || !assets + let ort: Ort | null = null + let session: import('onnxruntime-web').InferenceSession | null = null + let sr: Tensor | null = null + const chans: ChannelVad[] = [] + const total = Math.max(1, Math.round(init.durationSec * init.sampleRate)) // for the progress bar + let processed = 0 + + const zeroState = (): Tensor => new ort!.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]) + + const ready = (async () => { + if (disabled) return + try { + ort = await import('onnxruntime-web') + ort.env.wasm.numThreads = 1 + ort.env.wasm.wasmPaths = assets!.wasmBase + const modelBytes = await (await fetch(assets!.modelUrl)).arrayBuffer() + session = await ort.InferenceSession.create(modelBytes, { executionProviders: ['wasm'] }) + sr = new ort.Tensor('int64', [16000n]) + for (let ch = 0; ch < nc; ch++) { + chans.push({ resampler: new Resampler16k(init.sampleRate), probs: [], h: zeroState(), c: zeroState() }) + } + } catch (e) { + disabled = true + console.warn('[silero-vad] disabled, falling back to energy VAD:', e) + } + })() + + return { + async pushSegment(seg: AudioSegment): Promise { + await ready + if (disabled) return + // Never let a VAD error abort the shared decode loop (which also drives spectrogram/ + // waveform); on failure, disable VAD and leave the energy fallback in place. + try { + for (let ch = 0; ch < nc; ch++) { + const c = chans[ch]! + const owned = seg.channels[ch]!.subarray(0, seg.ownedSamples) + for (const frame of c.resampler.push(owned)) { + const input = new ort!.Tensor('float32', frame, [1, frame.length]) + const out = await session!.run({ input, h: c.h, c: c.c, sr: sr! }) + c.h = out['hn']!; c.c = out['cn']! + c.probs.push(out['output']!.data[0] as number) + } + } + } catch (e) { + disabled = true + console.warn('[silero-vad] inference error, disabling VAD:', e) + } + processed += seg.ownedSamples + post({ type: 'vadProgress', done: Math.min(processed, total), total }) + }, + + async finish(): Promise { + await ready + post({ type: 'vadProgress', done: total, total }) // always clear the bar (even if disabled) + // Post even when disabled (empty) so the caller's "computing" state clears. + if (disabled) { post({ type: 'vad', segments: [] }); return } + // Cache the per-frame probs so a later settings change re-segments instantly. + _probCache = chans.map(c => c.probs) + post({ type: 'vad', segments: mergeVadSegments(chans.map(c => segmentProbs(c.probs, settings))) }) + }, + } + }, +} diff --git a/packages/media-player/src/plugins/signal/spectrogram.ts b/packages/media-player/src/plugins/signal/spectrogram.ts index 9eb68c8..7c3b768 100644 --- a/packages/media-player/src/plugins/signal/spectrogram.ts +++ b/packages/media-player/src/plugins/signal/spectrogram.ts @@ -3,12 +3,13 @@ import type { SpectrogramTile } from '@mumo/timeline' const SPEC_DB_FLOOR = -160 const SPEC_DB_RANGE = 160 import type { SpectrogramSettings } from '../../types.js' -import { PREVIEW_SPEC_SETTINGS } from '../../types.js' -import type { SignalPlugin, AudioCtx, SignalPost } from './SignalPlugin.js' -import { runVadForAllChannels } from './vad.js' +import { SPEC_FREQ_HEADROOM } from '../../types.js' +import type { SignalPlugin, SignalRun, StreamInit, AudioSegment, SignalPost } from './SignalPlugin.js' let wasmSB: typeof WasmSampleBuffer | null = null export function setSampleBuffer(sb: typeof WasmSampleBuffer): void { wasmSB = sb } +/** The loaded Rust/WASM SampleBuffer class (null until the worker loads it), shared with the pitch plugin. */ +export function getSampleBuffer(): typeof WasmSampleBuffer | null { return wasmSB } const TILE_FRAMES = 2048 const OVERVIEW_MAX_WIDTH = 4096 @@ -20,15 +21,24 @@ function windowCode(w: SpectrogramSettings['window']): number { return 0 } -function nearestPow2(n: number): number { - const lower = Math.pow(2, Math.floor(Math.log2(Math.max(n, 1)))) - const upper = lower * 2 - return upper - n < n - lower ? upper : lower +function nextPow2(n: number): number { + return Math.pow(2, Math.ceil(Math.log2(Math.max(n, 1)))) } +// Praat framing (`Sound_to_Spectrogram`): the user's `windowLengthSec` is the *effective* width; a +// Gaussian window is physically **twice** that (so its tails aren't chopped), other windows 1×. The +// physical window (rounded to an even sample count) is then **zero-padded up to a power-of-two FFT** +// (Praat pads up; we used to round to the *nearest* pow2, which truncated the window). The extra +// frequency interpolation + the wider Gaussian is what gives Praat its smooth, harmonic-resolving look. function toSamples(settings: SpectrogramSettings, sampleRate: number) { + const effSamples = settings.windowLengthSec * sampleRate + const physical = settings.window === 'gaussian' ? 2 * effSamples : effSamples + let physicalWindowSize = Math.max(2, Math.round(physical)) + if (physicalWindowSize & 1) physicalWindowSize += 1 // even, as Praat makes nsamp_window even + const fftSize = Math.max(nextPow2(physicalWindowSize), 2) return { - windowSize: nearestPow2(settings.windowLengthSec * sampleRate), + physicalWindowSize, + fftSize, hop: Math.max(1, Math.round(settings.hopSec * sampleRate)), } } @@ -60,16 +70,20 @@ function fft(re: Float64Array, im: Float64Array): void { } } +// Window over `size` (physical) samples, using Praat's centered phase `p = (i − (n−1)/2)/n ∈ [−½, ½]`. +// Gaussian is Praat's truncated form `(exp(−48p²) − e⁻¹²)/(1 − e⁻¹²)` (≈0 at the physical edges), +// paired with the 2×-effective physical width from toSamples so the Gaussian is genuinely un-chopped. function buildWindow(size: number, kind: SpectrogramSettings['window']): Float32Array { const w = new Float32Array(size) + const edge = Math.exp(-12), gnorm = 1 / (1 - edge) for (let i = 0; i < size; i++) { + const p = (i - (size - 1) / 2) / size // [-0.5, 0.5] if (kind === 'hamming') { - w[i] = 0.54 - 0.46 * Math.cos((2 * Math.PI * i) / (size - 1)) + w[i] = 0.54 + 0.46 * Math.cos(2 * Math.PI * p) } else if (kind === 'gaussian') { - const half = size / 2, t = (i - half) / half - w[i] = Math.exp(-Math.PI * t * t) + w[i] = (Math.exp(-48 * p * p) - edge) * gnorm } else { - w[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (size - 1)) + w[i] = 0.5 + 0.5 * Math.cos(2 * Math.PI * p) // Hann } } return w @@ -96,33 +110,34 @@ function buildMelFilterbankJS(sampleRate: number, numLinearBins: number, melBand }) } -function applyMelJS(re: Float64Array, im: Float64Array, filterbank: MelBand[], curMag: Float32Array): void { +function applyMelJS(re: Float64Array, im: Float64Array, filterbank: MelBand[], curMag: Float32Array, energyScale = 1): void { for (let m = 0; m < filterbank.length; m++) { let energy = 0 for (const [k, w] of filterbank[m]!) { const r = re[k]!, im_ = im[k]! energy += (r * r + im_ * im_) * w } - curMag[m] = 10 * Math.log10(energy + 1e-20) + curMag[m] = 10 * Math.log10(energy * energyScale + 1e-20) } } function computeSpectrogramStatsJS( samples: Float32Array, sampleRate: number, - windowSize: number, + fftSize: number, hop: number, maxFreqHz: number, dynamicRangeDb: number, - win: Float32Array, + win: Float32Array, // win.length = physical window; frame is zero-padded to fftSize melFilterbank: MelBand[] | null, onProgress?: (done: number) => void, ) { maxFreqHz = Math.min(maxFreqHz, sampleRate / 2) - const maxBin = Math.round(maxFreqHz / (sampleRate / windowSize)) - const numLinearBins = Math.min(maxBin, windowSize / 2) + const winLen = win.length + const maxBin = Math.round(maxFreqHz / (sampleRate / fftSize)) + const numLinearBins = Math.min(maxBin, fftSize / 2) const numFreqBins = melFilterbank ? melFilterbank.length : numLinearBins - const numFrames = Math.max(1, Math.floor((samples.length - windowSize) / hop) + 1) + const numFrames = Math.max(1, Math.floor((samples.length - fftSize) / hop) + 1) const overviewWidth = Math.min(numFrames, OVERVIEW_MAX_WIDTH) const overviewBinSize = Math.ceil(numFrames / overviewWidth) const overviewAccum = new Float64Array(overviewWidth * numFreqBins) @@ -131,14 +146,14 @@ function computeSpectrogramStatsJS( const bandFlux = new Float32Array(numFrames * NUM_SNAP_BANDS) const frameRMS = new Float32Array(numFrames) let globalMin = Infinity, globalMax = -Infinity - const re = new Float64Array(windowSize), im = new Float64Array(windowSize) + const re = new Float64Array(fftSize), im = new Float64Array(fftSize) const prevMag = new Float32Array(numFreqBins), curMag = new Float32Array(numFreqBins) const framePeakDb = new Float32Array(numFrames) for (let f = 0; f < numFrames; f++) { const frameStart = f * hop re.fill(0); im.fill(0) - for (let j = 0; j < windowSize; j++) { + for (let j = 0; j < winLen; j++) { const idx = frameStart + j re[j] = idx < samples.length ? (samples[idx]! * win[j]!) : 0 } @@ -196,37 +211,64 @@ function computeSpectrogramStatsJS( return { numFreqBins, numLinearBins, numFrames, overviewRawDb, overviewWidth, flux, bandFlux, frameRMS } } +// Per-bin pre-emphasis in dB (Praat: `dbPerOct·log2(f/1000)`, 0 dB at 1 kHz, boosting highs). The +// DC/low bins get a large negative term (as in Praat, killing the DC component). null if disabled. +function buildPreEmphasis( + dbPerOct: number, numFreqBins: number, melFb: MelBand[] | null, + sampleRate: number, windowSize: number, maxFreqHz: number, +): Float32Array | null { + if (!dbPerOct) return null + const out = new Float32Array(numFreqBins) + const binHz = sampleRate / windowSize + const hzToMel = (hz: number) => 2595 * Math.log10(1 + hz / 700) + const melToHz = (mel: number) => 700 * (Math.pow(10, mel / 2595) - 1) + const maxMel = hzToMel(Math.min(maxFreqHz, sampleRate / 2)) + for (let k = 0; k < numFreqBins; k++) { + const freq = melFb ? melToHz((maxMel * (k + 0.5)) / numFreqBins) : k * binHz + out[k] = dbPerOct * (Math.log(Math.max(0, freq) / 1000 + 1e-308) / Math.LN2) + } + return out +} + // Produce a rawDb tile: one uint8 per (freq-bin, frame), quantised to SPEC_DB_FLOOR..+SPEC_DB_RANGE. // The Timeline applies a per-viewport LUT when uploading to GPU. function renderDetailTileRawDb( samples: Float32Array, - windowSize: number, hop: number, win: Float32Array, + fftSize: number, hop: number, win: Float32Array, // win.length = physical window; rest zero-padded numLinearBins: number, numFreqBins: number, startFrame: number, endFrame: number, melFilterbank: MelBand[] | null, + preEmphDb: Float32Array | null, // per-bin dB to add (Praat pre-emphasis); null = none ): Uint8Array { const tileWidth = endFrame - startFrame + const winLen = win.length + // Normalize by the window's coherent gain (Σw) so magnitudes are independent of window length — + // a full-scale sinusoid reads ~0 dB regardless of window size (Praat normalizes power by windowssq). + let winSum = 0 + for (let j = 0; j < winLen; j++) winSum += win[j]! + const invWinSum = winSum > 0 ? 1 / winSum : 1 const rawDb = new Uint8Array(tileWidth * numFreqBins) - const re = new Float64Array(windowSize), im = new Float64Array(windowSize) + const re = new Float64Array(fftSize), im = new Float64Array(fftSize) const curMag = new Float32Array(numFreqBins) for (let f = startFrame; f < endFrame; f++) { const frameStart = f * hop re.fill(0); im.fill(0) - for (let j = 0; j < windowSize; j++) { + for (let j = 0; j < winLen; j++) { const idx = frameStart + j re[j] = idx < samples.length ? (samples[idx]! * win[j]!) : 0 } fft(re, im) if (melFilterbank) { - applyMelJS(re, im, melFilterbank, curMag) + applyMelJS(re, im, melFilterbank, curMag, invWinSum * invWinSum) } else { for (let k = 0; k < numLinearBins; k++) { - curMag[k] = 20 * Math.log10(Math.sqrt(re[k]! * re[k]! + im[k]! * im[k]!) + 1e-10) + curMag[k] = 20 * Math.log10(Math.sqrt(re[k]! * re[k]! + im[k]! * im[k]!) * invWinSum + 1e-10) } } const localF = f - startFrame for (let k = 0; k < numFreqBins; k++) { - const q = Math.max(0, Math.min(255, Math.round((curMag[k]! - SPEC_DB_FLOOR) / SPEC_DB_RANGE * 255))) + const db = curMag[k]! + (preEmphDb ? preEmphDb[k]! : 0) + const q = Math.max(0, Math.min(255, Math.round((db - SPEC_DB_FLOOR) / SPEC_DB_RANGE * 255))) rawDb[(numFreqBins - 1 - k) * tileWidth + localF] = q } } @@ -300,181 +342,187 @@ function detectFluxPeaks(flux: Float32Array, sampleRate: number, hop: number) { return { timestamps: new Float32Array(events.map(e => e.time)), strengths: new Float32Array(events.map(e => e.strength)) } } -type RunResult = { flux: Float32Array; bandFlux: Float32Array; numSnapBands: number; frameRMS: Float32Array } - -function runChannelWasm( - samples: Float32Array, sampleRate: number, duration: number, - settings: SpectrogramSettings, channelIndex: number, - doneRef: { value: number }, totalFrames: number, - post: SignalPost, -): RunResult { - const SB = wasmSB - if (!SB) throw new Error('WASM not loaded') - const { windowSize, hop } = toSamples(settings, sampleRate) - const buf = new SB(samples, windowSize, windowCode(settings.window)) - try { - const base = doneRef.value - const melBands = settings.scale === 'mel' ? settings.melBands : 0 - const stats = buf.compute_stats(hop, settings.maxFreqHz, sampleRate, settings.dynamicRangeDb, melBands, (done: number, _total: number) => { - doneRef.value = base + done - post({ type: 'progress', done: doneRef.value, total: totalFrames }) - }) - const { num_freq_bins: numFreqBins, num_frames: numFrames, num_snap_bands: numSnapBands } = stats - // Discard WASM overview pixels — we build a rawDb overview from JS tiles instead. - stats.take_overview_pixels() - const flux = new Float32Array(stats.take_flux()) - const bandFlux = new Float32Array(stats.take_band_flux()) - const frameRMS = new Float32Array(stats.take_frame_rms()) - stats.free() - - doneRef.value = base + numFrames - post({ type: 'progress', done: doneRef.value, total: totalFrames }) - - // Emit rawDb detail tiles and accumulate into overview simultaneously. - const win = buildWindow(windowSize, settings.window) - const maxBin = Math.round(Math.min(settings.maxFreqHz, sampleRate / 2) / (sampleRate / windowSize)) - const numLinearBinsJS = Math.min(maxBin, windowSize / 2) - const melFbJS = settings.scale === 'mel' - ? buildMelFilterbankJS(sampleRate, numLinearBinsJS, settings.melBands, settings.maxFreqHz) - : null - const hopDuration = hop / sampleRate, tileCount = Math.ceil(numFrames / TILE_FRAMES) - const ovWidth = Math.min(numFrames, OVERVIEW_MAX_WIDTH) - const ovBinSize = Math.ceil(numFrames / ovWidth) - const ovAccum = new Float32Array(ovWidth * numFreqBins) - const ovCount = new Uint32Array(ovWidth) - const p2Base = doneRef.value - for (let t = 0; t < tileCount; t++) { - const sf = t * TILE_FRAMES, ef = Math.min((t + 1) * TILE_FRAMES, numFrames) - const rawDb = renderDetailTileRawDb(samples, windowSize, hop, win, numLinearBinsJS, numFreqBins, sf, ef, melFbJS) - // Accumulate into overview (work in rawDb units to avoid float↔dB conversions) - for (let f = sf; f < ef; f++) { - const ob = Math.min(Math.floor(f / ovBinSize), ovWidth - 1) - const localF = f - sf - for (let k = 0; k < numFreqBins; k++) { - const idx = ob * numFreqBins + k - ovAccum[idx] = (ovAccum[idx]! + rawDb[(numFreqBins - 1 - k) * (ef - sf) + localF]!) - } - ovCount[ob] = ovCount[ob]! + 1 - } - const tile: SpectrogramTile = { tileIndex: t, rawDb, width: ef - sf, height: numFreqBins, timeStart: sf * hopDuration, timeEnd: t === tileCount - 1 ? duration : ef * hopDuration } - doneRef.value = p2Base + ef - post({ type: 'progress', done: doneRef.value, total: totalFrames }) - post({ type: 'spectrogramTile', channelIndex, tile }, [rawDb.buffer]) - } - // Build and send the overview after all tiles so it uses the same rawDb encoding. - const overviewRawDb = new Uint8Array(ovWidth * numFreqBins) - for (let x = 0; x < ovWidth; x++) { - const cnt = ovCount[x] || 1 - for (let k = 0; k < numFreqBins; k++) - overviewRawDb[(numFreqBins - 1 - k) * ovWidth + x] = Math.round(ovAccum[x * numFreqBins + k]! / cnt) - } - post({ type: 'spectrogramOverview', channelIndex, tile: { tileIndex: -1, rawDb: overviewRawDb, width: ovWidth, height: numFreqBins, timeStart: 0, timeEnd: duration } }, [overviewRawDb.buffer]) - - return { flux, bandFlux, numSnapBands, frameRMS } - } finally { - buf.free() - } +// --------------------------------------------------------------------------- +// Streaming run — the file is decoded in ~1-2 min segments (see the worker) so the +// full-file PCM is never resident. Each segment computes its own flux/tiles; the +// overview and per-frame onset features accumulate across segments and are emitted at +// finish. Detail-tile pixels use a fixed dB encoding, so segments are self-contained +// (no cross-segment normalization). Flux at each segment's first frame is 0 (no prior +// frame carried across the boundary) — a negligible artifact every ~90s. +// --------------------------------------------------------------------------- + +/** Frame/tile grid params the worker needs to size segments to the spectrogram grid. */ +export function specFrameParams(settings: SpectrogramSettings, sampleRate: number): { hop: number; windowSize: number; tileFrames: number } { + const { fftSize, hop } = toSamples(settings, sampleRate) + // The segmenter must provide fftSize samples per frame (the JS tiles read only the physical window + // and zero-pad, but the WASM stats path reads the full fftSize span). + return { hop, windowSize: fftSize, tileFrames: TILE_FRAMES } } -function runChannelJS( - samples: Float32Array, sampleRate: number, duration: number, - settings: SpectrogramSettings, channelIndex: number, - doneRef: { value: number }, totalFrames: number, - post: SignalPost, -): RunResult { - const { windowSize, hop } = toSamples(settings, sampleRate) - const win = buildWindow(windowSize, settings.window) - const p1Base = doneRef.value - const maxBin = Math.round(Math.min(settings.maxFreqHz, sampleRate / 2) / (sampleRate / windowSize)) - const numLinearBinsJS = Math.min(maxBin, windowSize / 2) - const melFilterbankJS = settings.scale === 'mel' - ? buildMelFilterbankJS(sampleRate, numLinearBinsJS, settings.melBands, settings.maxFreqHz) - : null - - const { numFreqBins, numLinearBins, numFrames: nf, overviewRawDb, overviewWidth, flux, bandFlux, frameRMS } - = computeSpectrogramStatsJS(samples, sampleRate, windowSize, hop, settings.maxFreqHz, settings.dynamicRangeDb, win, melFilterbankJS, done => { - doneRef.value = p1Base + done - post({ type: 'progress', done: doneRef.value, total: totalFrames }) - }) - - doneRef.value = p1Base + nf - post({ type: 'spectrogramOverview', channelIndex, tile: { tileIndex: -1, rawDb: overviewRawDb, width: overviewWidth, height: numFreqBins, timeStart: 0, timeEnd: duration } }, [overviewRawDb.buffer]) - - const hopDuration = hop / sampleRate, tileCount = Math.ceil(nf / TILE_FRAMES), p2Base = doneRef.value - for (let t = 0; t < tileCount; t++) { - const sf = t * TILE_FRAMES, ef = Math.min((t + 1) * TILE_FRAMES, nf) - const rawDb = renderDetailTileRawDb(samples, windowSize, hop, win, numLinearBins, numFreqBins, sf, ef, melFilterbankJS) - const tile: SpectrogramTile = { tileIndex: t, rawDb, width: ef - sf, height: numFreqBins, timeStart: sf * hopDuration, timeEnd: t === tileCount - 1 ? duration : ef * hopDuration } - doneRef.value = p2Base + ef - post({ type: 'progress', done: doneRef.value, total: totalFrames }) - post({ type: 'spectrogramTile', channelIndex, tile }, [rawDb.buffer]) +const NOOP_PROGRESS = (() => { /* per-segment progress is reported by the run */ }) as unknown as (done: number, total: number) => void + +// Per-segment flux / band-flux / frame-RMS. WASM when available (native), else JS fallback. +function computeSegmentStats( + samples: Float32Array, sampleRate: number, settings: SpectrogramSettings, + fftSize: number, hop: number, win: Float32Array, melFb: MelBand[] | null, +): { flux: Float32Array; bandFlux: Float32Array; frameRMS: Float32Array; numSnapBands: number } { + if (wasmSB) { + const buf = new wasmSB(samples, fftSize, windowCode(settings.window)) + try { + const melBands = settings.scale === 'mel' ? settings.melBands : 0 + const stats = buf.compute_stats(hop, settings.maxFreqHz, sampleRate, settings.dynamicRangeDb, melBands, NOOP_PROGRESS) + const numSnapBands = stats.num_snap_bands + stats.take_overview_pixels() // discard — overview is built from detail tiles + const flux = new Float32Array(stats.take_flux()) + const bandFlux = new Float32Array(stats.take_band_flux()) + const frameRMS = new Float32Array(stats.take_frame_rms()) + stats.free() + return { flux, bandFlux, frameRMS, numSnapBands } + } finally { + buf.free() + } } - - return { flux, bandFlux, numSnapBands: NUM_SNAP_BANDS, frameRMS } + const r = computeSpectrogramStatsJS(samples, sampleRate, fftSize, hop, settings.maxFreqHz, settings.dynamicRangeDb, win, melFb) + return { flux: r.flux, bandFlux: r.bandFlux, frameRMS: r.frameRMS, numSnapBands: NUM_SNAP_BANDS } } -function runPreview(ctx: AudioCtx, post: SignalPost): void { - const { channels, sampleRate, duration, settings } = ctx - const { windowSize, hop } = toSamples(PREVIEW_SPEC_SETTINGS, sampleRate) - const { maxFreqHz, dynamicRangeDb } = PREVIEW_SPEC_SETTINGS - const melBands = settings.scale === 'mel' ? settings.melBands : 0 - const win = buildWindow(windowSize, PREVIEW_SPEC_SETTINGS.window) - - for (let ch = 0; ch < channels.length; ch++) { - const samples = channels[ch]! - const maxBin = Math.round(Math.min(maxFreqHz, sampleRate / 2) / (sampleRate / windowSize)) - const numLinearBins = Math.min(maxBin, windowSize / 2) - const melFb = melBands > 0 ? buildMelFilterbankJS(sampleRate, numLinearBins, melBands, maxFreqHz) : null - const { overviewRawDb, overviewWidth, numFreqBins } = computeSpectrogramStatsJS(samples, sampleRate, windowSize, hop, maxFreqHz, dynamicRangeDb, win, melFb) - post( - { type: 'spectrogramOverview', channelIndex: ch, tile: { tileIndex: -1, rawDb: overviewRawDb, width: overviewWidth, height: numFreqBins, timeStart: 0, timeEnd: duration } }, - [overviewRawDb.buffer], - ) - } +interface ChannelAccum { + ovAccum: Float64Array + ovCount: Uint32Array + flux: Float32Array + frameRMS: Float32Array + bandFlux: Float32Array | null // lazily sized once numSnapBands is known + numSnapBands: number } export const spectrogramPlugin: SignalPlugin = { id: 'spectrogram', - async analyze(ctx: AudioCtx, post: SignalPost): Promise { - const { channels, sampleRate, duration, settings, trigger } = ctx - const { windowSize, hop } = toSamples(settings, sampleRate) - const sampleLen = channels[0]!.length - const numFrames = Math.max(1, Math.floor((sampleLen - windowSize) / hop) + 1) - const totalFrames = numFrames * channels.length * 2 + createRun(init: StreamInit, post: SignalPost): SignalRun { + const { sampleRate, channelCount, durationSec, settings } = init + const { physicalWindowSize, fftSize, hop } = toSamples(settings, sampleRate) + // Store bins up to a headroom ceiling (linear scale only) so the displayed frequency window can + // be narrowed — or widened up to this ceiling — at render time without re-decoding. Mel bins are + // non-uniform in Hz, so mel stores exactly maxFreqHz (frequency changes recompute). + const storedMaxFreqHz = settings.scale === 'mel' + ? settings.maxFreqHz + : Math.min(sampleRate / 2, settings.maxFreqHz * SPEC_FREQ_HEADROOM) + const binHz = sampleRate / fftSize + const maxBin = Math.round(Math.min(storedMaxFreqHz, sampleRate / 2) / binHz) + const numLinearBins = Math.min(maxBin, fftSize / 2) + const melFb = settings.scale === 'mel' + ? buildMelFilterbankJS(sampleRate, numLinearBins, settings.melBands, settings.maxFreqHz) + : null + const numFreqBins = melFb ? melFb.length : numLinearBins + // Actual top frequency the stored rows cover (quantised to a bin edge for linear). + const tileMaxFreqHz = melFb ? settings.maxFreqHz : numLinearBins * binHz + const win = buildWindow(physicalWindowSize, settings.window) + + // Praat-style pre-emphasis: +preEmphasisDbPerOct·log2(f/1000) baked per frequency bin (0 = off). + const preEmphDb = buildPreEmphasis(settings.preEmphasisDbPerOct, numFreqBins, melFb, sampleRate, fftSize, tileMaxFreqHz) + + // Frames span fftSize samples (matches the segmenter's per-frame provision); the physical window + // is windowed into the zero-padded fftSize buffer inside renderDetailTileRawDb. + const totalSamples = Math.max(fftSize, Math.round(durationSec * sampleRate)) + const totalFrames = Math.max(1, Math.floor((totalSamples - fftSize) / hop) + 1) + const ovWidth = Math.min(totalFrames, OVERVIEW_MAX_WIDTH) + const ovBinSize = Math.ceil(totalFrames / ovWidth) + const hopDuration = hop / sampleRate + // totalFrames is estimated from the container duration; the actual decoded frame count + // (filledFrames) can be smaller (containers often over-report audio length). Progress and + // the overview extent are reconciled to filledFrames at finish so the overview isn't + // stretched across a grid wider than the real content. + const totalProgress = totalFrames const doneRef = { value: 0 } - const frameRMSCache: Float32Array[] = [] - - if (trigger !== 'reanalyze') runPreview(ctx, post) - - for (let ch = 0; ch < channels.length; ch++) { - const samples = channels[ch]! - let result: RunResult - - if (wasmSB) { - result = runChannelWasm(samples, sampleRate, duration, settings, ch, doneRef, totalFrames, post) - } else { - result = runChannelJS(samples, sampleRate, duration, settings, ch, doneRef, totalFrames, post) - } - - const { flux, bandFlux, numSnapBands, frameRMS } = result - frameRMSCache[ch] = frameRMS - - const { timestamps, strengths } = computeOnsets(flux, frameRMS, sampleRate, hop) - const bandTimestamps: Float32Array[] = [], bandStrengths: Float32Array[] = [] - for (let b = 0; b < numSnapBands; b++) { - const slice = new Float32Array(flux.length) - for (let f = 0; f < flux.length; f++) slice[f] = bandFlux[f * numSnapBands + b]! - const { timestamps: bt, strengths: bs } = detectFluxPeaks(slice, sampleRate, hop) - bandTimestamps.push(bt); bandStrengths.push(bs) - } - - const transfer = [timestamps.buffer, strengths.buffer, ...bandTimestamps.map(a => a.buffer), ...bandStrengths.map(a => a.buffer)] - post({ type: 'onsets', channelIndex: ch, timestamps, strengths, bandTimestamps, bandStrengths }, transfer) + let filledFrames = 0 + + const acc: ChannelAccum[] = Array.from({ length: channelCount }, () => ({ + ovAccum: new Float64Array(ovWidth * numFreqBins), + ovCount: new Uint32Array(ovWidth), + flux: new Float32Array(totalFrames), + frameRMS: new Float32Array(totalFrames), + bandFlux: null, + numSnapBands: NUM_SNAP_BANDS, + })) + + return { + pushSegment(seg: AudioSegment): void { + const F0 = seg.firstFrame + const fc = Math.min(seg.frameCount, totalFrames - F0) + if (fc <= 0) return + for (let ch = 0; ch < channelCount; ch++) { + const samples = seg.channels[ch]! + const a = acc[ch]! + const { flux, bandFlux, frameRMS, numSnapBands } = computeSegmentStats(samples, sampleRate, settings, fftSize, hop, win, melFb) + if (!a.bandFlux) { a.bandFlux = new Float32Array(totalFrames * numSnapBands); a.numSnapBands = numSnapBands } + const nsb = a.numSnapBands + a.flux.set(flux.subarray(0, fc), F0) + a.frameRMS.set(frameRMS.subarray(0, fc), F0) + a.bandFlux.set(bandFlux.subarray(0, fc * nsb), F0 * nsb) + + const tileCount = Math.ceil(fc / TILE_FRAMES) + for (let t = 0; t < tileCount; t++) { + const sf = t * TILE_FRAMES, ef = Math.min((t + 1) * TILE_FRAMES, fc) + const tw = ef - sf + const rawDb = renderDetailTileRawDb(samples, fftSize, hop, win, numLinearBins, numFreqBins, sf, ef, melFb, preEmphDb) + for (let lf = sf; lf < ef; lf++) { + const g = F0 + lf + const ob = Math.min(Math.floor(g / ovBinSize), ovWidth - 1) + const col = lf - sf + for (let k = 0; k < numFreqBins; k++) { + a.ovAccum[ob * numFreqBins + k] = a.ovAccum[ob * numFreqBins + k]! + rawDb[(numFreqBins - 1 - k) * tw + col]! + } + a.ovCount[ob] = a.ovCount[ob]! + 1 + } + const globalTile = (F0 / TILE_FRAMES) + t + const timeStart = (F0 + sf) * hopDuration + const timeEnd = (F0 + ef) * hopDuration + const tile: SpectrogramTile = { tileIndex: globalTile, rawDb, width: tw, height: numFreqBins, timeStart, timeEnd, maxFreqHz: tileMaxFreqHz } + post({ type: 'spectrogramTile', channelIndex: ch, tile }, [rawDb.buffer]) + } + } + filledFrames = Math.max(filledFrames, F0 + fc) + doneRef.value += fc + post({ type: 'progress', done: doneRef.value, total: totalProgress }) + }, + + finish(): void { + // Reconcile to the frames actually decoded: the overview covers [0, nActual) frames, so + // emit only the columns those frames touched and place it at the true content end time. + const nActual = Math.max(1, filledFrames) + const ovUsed = Math.min(ovWidth, Math.max(1, Math.ceil(nActual / ovBinSize))) + const contentEndSec = nActual * hopDuration + for (let ch = 0; ch < channelCount; ch++) { + const a = acc[ch]! + const nsb = a.numSnapBands + const bandFlux = a.bandFlux ?? new Float32Array(totalFrames * nsb) + + const { timestamps, strengths } = computeOnsets(a.flux, a.frameRMS, sampleRate, hop) + const bandTimestamps: Float32Array[] = [], bandStrengths: Float32Array[] = [] + for (let b = 0; b < nsb; b++) { + const slice = new Float32Array(totalFrames) + for (let f = 0; f < totalFrames; f++) slice[f] = bandFlux[f * nsb + b]! + const { timestamps: bt, strengths: bs } = detectFluxPeaks(slice, sampleRate, hop) + bandTimestamps.push(bt); bandStrengths.push(bs) + } + const transfer = [timestamps.buffer, strengths.buffer, ...bandTimestamps.map(x => x.buffer), ...bandStrengths.map(x => x.buffer)] + post({ type: 'onsets', channelIndex: ch, timestamps, strengths, bandTimestamps, bandStrengths }, transfer) + + const overviewRawDb = new Uint8Array(ovUsed * numFreqBins) + for (let x = 0; x < ovUsed; x++) { + const cnt = a.ovCount[x] || 1 + for (let k = 0; k < numFreqBins; k++) { + overviewRawDb[(numFreqBins - 1 - k) * ovUsed + x] = Math.round(a.ovAccum[x * numFreqBins + k]! / cnt) + } + } + post({ type: 'spectrogramOverview', channelIndex: ch, tile: { tileIndex: -1, rawDb: overviewRawDb, width: ovUsed, height: numFreqBins, timeStart: 0, timeEnd: contentEndSec, maxFreqHz: tileMaxFreqHz } }, [overviewRawDb.buffer]) + } + // VAD is produced by the Silero plugin (sileroVad.ts); the spectrogram no longer emits an + // energy-VAD fallback. + // Container over-report leaves doneRef below totalFrames; force the bar to clear. + post({ type: 'progress', done: totalProgress, total: totalProgress }) + }, } - - const segments = await runVadForAllChannels(channels, sampleRate, frameRMSCache, hop) - if (segments.length > 0) post({ type: 'vad', segments }) }, } diff --git a/packages/media-player/src/plugins/signal/vad.ts b/packages/media-player/src/plugins/signal/vad.ts index 8d4e084..56b3b36 100644 --- a/packages/media-player/src/plugins/signal/vad.ts +++ b/packages/media-player/src/plugins/signal/vad.ts @@ -54,35 +54,11 @@ export function computeEnergyVad( } -async function runSileroVad(samples: Float32Array, sampleRate: number): Promise { - try { - const { NonRealTimeVAD } = await import('@ricky0123/vad-web') - const vad = await NonRealTimeVAD.new({ - modelURL: new URL('./silero_vad_legacy.onnx', import.meta.url).href, - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access - ortConfig: (ort: any) => { ort.env.wasm.numThreads = 1; ort.env.wasm.wasmPaths = new URL('./', import.meta.url).href }, - }) - const segments: VadSegment[] = [] - for await (const { start, end } of vad.run(samples, sampleRate)) { - segments.push({ start: start / sampleRate, end: end / sampleRate }) - } - return segments - } catch { - return null - } -} - -export async function runVadForAllChannels( - channels: Float32Array[], - sampleRate: number, - frameRMSCache: Float32Array[], - hop: number, -): Promise { +/** Merge per-channel VAD segments into a single "is anyone speaking" timeline: sort by start + * and coalesce segments that touch or overlap (within 50 ms). Used by the Silero VAD plugin. */ +export function mergeVadSegments(perChannel: VadSegment[][]): VadSegment[] { const all: VadSegment[] = [] - for (let ch = 0; ch < channels.length; ch++) { - const result = await runSileroVad(channels[ch]!, sampleRate) - all.push(...(result ?? computeEnergyVad(frameRMSCache[ch]!, sampleRate, hop))) - } + for (const segs of perChannel) all.push(...segs) all.sort((a, b) => a.start - b.start) const merged: VadSegment[] = [] for (const seg of all) { diff --git a/packages/media-player/src/plugins/signal/waveform.ts b/packages/media-player/src/plugins/signal/waveform.ts index aa2580b..20311c6 100644 --- a/packages/media-player/src/plugins/signal/waveform.ts +++ b/packages/media-player/src/plugins/signal/waveform.ts @@ -1,46 +1,57 @@ import type { WaveformBins } from '@mumo/timeline' -import type { SignalPlugin, AudioCtx, SignalPost } from './SignalPlugin.js' +import type { SignalPlugin, SignalRun, StreamInit, AudioSegment, SignalPost } from './SignalPlugin.js' const MS_PER_BIN = 5 -function computeWaveform(samples: Float32Array, sampleRate: number): WaveformBins { - const numBins = Math.min(200_000, Math.max(100, Math.ceil(samples.length / sampleRate / (MS_PER_BIN / 1000)))) - const binSize = Math.max(1, Math.floor(samples.length / numBins)) - const binCount = Math.floor(samples.length / binSize) - const peakPos = new Float32Array(binCount) - const peakNeg = new Float32Array(binCount) - const rms = new Float32Array(binCount) - - for (let b = 0; b < binCount; b++) { - let pk = 0, pn = 0, sq = 0 - const start = b * binSize, end = start + binSize - for (let i = start; i < end; i++) { - const v = samples[i]! - if (v > pk) pk = v - if (v < pn) pn = v - sq += v * v - } - peakPos[b] = pk - peakNeg[b] = pn - rms[b] = Math.sqrt(sq / binSize) - } - - return { peakPos, peakNeg, rms, binDuration: binSize / sampleRate, binCount } -} - +// Streaming min/max/RMS binning. Bin size is derived up front from the media duration; each +// segment's owned samples are folded into the running bin, and a partial bin carries across +// segment boundaries. Bins are held (a few MB at most) and emitted once, at finish. export const waveformPlugin: SignalPlugin = { id: 'waveform', - // eslint-disable-next-line @typescript-eslint/require-await - async analyze({ channels, sampleRate, trigger }: AudioCtx, post: SignalPost) { - if (trigger === 'reanalyze') return + createRun(init: StreamInit, post: SignalPost): SignalRun { + const active = init.trigger !== 'reanalyze' // waveform is independent of spectrogram settings + const nc = init.channelCount + const totalSamples = Math.max(1, Math.round(init.durationSec * init.sampleRate)) + const numBins = Math.min(200_000, Math.max(100, Math.ceil(totalSamples / init.sampleRate / (MS_PER_BIN / 1000)))) + const binSize = Math.max(1, Math.floor(totalSamples / numBins)) + const maxBins = Math.floor(totalSamples / binSize) + 1 + + const peakPos = Array.from({ length: nc }, () => new Float32Array(maxBins)) + const peakNeg = Array.from({ length: nc }, () => new Float32Array(maxBins)) + const rms = Array.from({ length: nc }, () => new Float32Array(maxBins)) + const st = Array.from({ length: nc }, () => ({ bin: 0, pk: 0, pn: 0, sq: 0, cnt: 0 })) + + return { + pushSegment(seg: AudioSegment): void { + if (!active) return + for (let ch = 0; ch < nc; ch++) { + const s = seg.channels[ch]!, c = st[ch]! + const owned = seg.ownedSamples + for (let i = 0; i < owned; i++) { + const v = s[i]! + if (v > c.pk) c.pk = v + if (v < c.pn) c.pn = v + c.sq += v * v + if (++c.cnt === binSize) { + if (c.bin < maxBins) { peakPos[ch]![c.bin] = c.pk; peakNeg[ch]![c.bin] = c.pn; rms[ch]![c.bin] = Math.sqrt(c.sq / binSize) } + c.bin++; c.pk = 0; c.pn = 0; c.sq = 0; c.cnt = 0 + } + } + } + }, - for (let ch = 0; ch < channels.length; ch++) { - const bins = computeWaveform(channels[ch]!, sampleRate) - post( - { type: 'waveform', channelIndex: ch, bins }, - [bins.peakPos.buffer, bins.peakNeg.buffer, bins.rms.buffer], - ) + finish(): void { + if (!active) return + for (let ch = 0; ch < nc; ch++) { + const binCount = Math.min(st[ch]!.bin, maxBins) + const pp = peakPos[ch]!.slice(0, binCount) + const pn = peakNeg[ch]!.slice(0, binCount) + const rr = rms[ch]!.slice(0, binCount) + const bins: WaveformBins = { peakPos: pp, peakNeg: pn, rms: rr, binDuration: binSize / init.sampleRate, binCount } + post({ type: 'waveform', channelIndex: ch, bins }, [pp.buffer, pn.buffer, rr.buffer]) + } + }, } }, } diff --git a/packages/media-player/src/segmenter.ts b/packages/media-player/src/segmenter.ts new file mode 100644 index 0000000..3a292e4 --- /dev/null +++ b/packages/media-player/src/segmenter.ts @@ -0,0 +1,90 @@ +import type { AudioSegment } from './plugins/signal/SignalPlugin.js' + +/** + * Buffers decoded PCM and cuts it into spectrogram-frame-aligned segments, each carrying a + * trailing window overlap so its last owned frame is complete. Bounded to roughly one segment + * of samples — the whole-file PCM is never resident. + * + * Frame F starts at absolute sample `F * hop`. A segment owns global frames `[firstFrame, + * firstFrame + frameCount)` and provides samples `[firstFrame*hop, lastFrame*hop + windowSize)` + * so every owned frame's window is complete; the trailing `windowSize - hop` samples are re-sent + * at the start of the next segment. `ownedSamples` marks the non-overlapping prefix. + */ +export class SegmentProducer { + private readonly chunks: Float32Array[][] + private bufStart = 0 // absolute sample index of chunks[ch][0][0] + private bufLen = 0 // samples buffered from bufStart + private nextFrame = 0 // firstFrame of the next segment + + constructor( + private readonly channelCount: number, + private readonly hop: number, + private readonly windowSize: number, + private readonly segFrames: number, + private readonly totalFrames: number, + ) { + this.chunks = Array.from({ length: channelCount }, () => []) + } + + add(perChannel: Float32Array[], frames: number): void { + for (let ch = 0; ch < this.channelCount; ch++) this.chunks[ch]!.push(perChannel[ch]!) + this.bufLen += frames + } + + private copyRange(ch: number, start: number, end: number): Float32Array { + const out = new Float32Array(end - start) + let pos = this.bufStart + for (const c of this.chunks[ch]!) { + const cStart = pos, cEnd = pos + c.length + const from = Math.max(start, cStart), to = Math.min(end, cEnd) + if (from < to) out.set(c.subarray(from - cStart, to - cStart), from - start) + pos = cEnd + if (pos >= end) break + } + return out + } + + private trimTo(newStart: number): void { + let drop = newStart - this.bufStart + while (drop > 0 && this.chunks[0]!.length > 0 && drop >= this.chunks[0]![0]!.length) { + const len = this.chunks[0]![0]!.length + for (let ch = 0; ch < this.channelCount; ch++) this.chunks[ch]!.shift() + drop -= len; this.bufStart += len; this.bufLen -= len + } + if (drop > 0 && this.chunks[0]!.length > 0) { + for (let ch = 0; ch < this.channelCount; ch++) this.chunks[ch]![0] = this.chunks[ch]![0]!.subarray(drop) + this.bufStart += drop; this.bufLen -= drop + } + } + + /** Return the next ready segment, or null if more data is needed (unless `final`). */ + tryCut(final: boolean): AudioSegment | null { + const F0 = this.nextFrame + if (F0 >= this.totalFrames) return null + const bufEnd = this.bufStart + this.bufLen + let F1 = Math.min(F0 + this.segFrames, this.totalFrames) + const segStart = F0 * this.hop + let neededEnd = (F1 - 1) * this.hop + this.windowSize + if (!final && bufEnd < neededEnd) return null + + let segEnd: number + if (bufEnd >= neededEnd) { + segEnd = neededEnd + } else { + // final drain with fewer samples than a full segment: use what's buffered + const avail = Math.floor((bufEnd - segStart - this.windowSize) / this.hop) + 1 + F1 = Math.min(F0 + Math.max(0, avail), this.totalFrames) + if (F1 <= F0) { this.nextFrame = this.totalFrames; return null } + neededEnd = (F1 - 1) * this.hop + this.windowSize + segEnd = Math.min(neededEnd, bufEnd) + } + + const frameCount = F1 - F0 + const isLast = F1 >= this.totalFrames + const ownedSamples = isLast ? segEnd - segStart : frameCount * this.hop + const channels = Array.from({ length: this.channelCount }, (_, ch) => this.copyRange(ch, segStart, segEnd)) + this.nextFrame = F1 + this.trimTo(F1 * this.hop) + return { channels, startSample: segStart, ownedSamples, firstFrame: F0, frameCount, isLast } + } +} diff --git a/packages/media-player/src/types.ts b/packages/media-player/src/types.ts index d9f6acd..e7f5973 100644 --- a/packages/media-player/src/types.ts +++ b/packages/media-player/src/types.ts @@ -5,25 +5,31 @@ export type { SpectrogramTile, WaveformBins } export interface SpectrogramSettings { windowLengthSec: number hopSec: number - maxFreqHz: number + maxFreqHz: number // upper frequency shown (display crop); also the analysis target + viewMinHz: number // lower frequency shown (display crop; 0 = DC) window: 'hann' | 'hamming' | 'gaussian' dynamicRangeDb: number gamma: number scale: 'linear' | 'mel' melBands: number monoMix: boolean + preEmphasisDbPerOct: number // per-frequency high-boost baked into the image (Praat: 6 dB/oct; 0 = off) } +// Analysis stores bins up to this multiple of maxFreqHz (linear scale only) so the frequency +// window can be narrowed (or widened up to the ceiling) at DISPLAY time without re-decoding. +export const SPEC_FREQ_HEADROOM = 1.5 + export const PREVIEW_SPEC_SETTINGS: SpectrogramSettings = { - windowLengthSec: 0.020, hopSec: 0.010, maxFreqHz: 8000, window: 'gaussian', dynamicRangeDb: 70, - gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false, + windowLengthSec: 0.020, hopSec: 0.010, maxFreqHz: 8000, viewMinHz: 0, window: 'gaussian', dynamicRangeDb: 70, + gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false, preEmphasisDbPerOct: 6.0, } export const SPEC_PRESETS = [ - { label: 'Wide-band', windowLengthSec: 0.005, hopSec: 0.0025, maxFreqHz: 5000, window: 'gaussian', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false }, - { label: 'Narrow-band', windowLengthSec: 0.020, hopSec: 0.010, maxFreqHz: 5000, window: 'gaussian', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false }, - { label: 'Broad-band', windowLengthSec: 0.010, hopSec: 0.005, maxFreqHz: 5500, window: 'gaussian', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false }, - { label: 'Full range', windowLengthSec: 0.010, hopSec: 0.005, maxFreqHz: 22050, window: 'hann', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false }, + { label: 'Wide-band', windowLengthSec: 0.005, hopSec: 0.0025, maxFreqHz: 5000, viewMinHz: 0, window: 'gaussian', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false, preEmphasisDbPerOct: 6.0 }, + { label: 'Narrow-band', windowLengthSec: 0.020, hopSec: 0.010, maxFreqHz: 5000, viewMinHz: 0, window: 'gaussian', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false, preEmphasisDbPerOct: 6.0 }, + { label: 'Broad-band', windowLengthSec: 0.010, hopSec: 0.005, maxFreqHz: 5500, viewMinHz: 0, window: 'gaussian', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false, preEmphasisDbPerOct: 6.0 }, + { label: 'Full range', windowLengthSec: 0.010, hopSec: 0.005, maxFreqHz: 22050, viewMinHz: 0, window: 'hann', dynamicRangeDb: 70, gamma: 1.2, scale: 'linear', melBands: 80, monoMix: false, preEmphasisDbPerOct: 6.0 }, ] as const satisfies Array<{ label: string } & SpectrogramSettings> export const DEFAULT_SPEC_SETTINGS: SpectrogramSettings = { ...SPEC_PRESETS[1] } @@ -50,27 +56,53 @@ export interface MediaTrack { export type WorkerRequest = | { - type: 'initStream' + // Decode + analyze a media file entirely inside the worker (off the main thread). + type: 'analyze' + url: string settings: SpectrogramSettings pluginSettings?: Record } | { - type: 'chunk' - channelData: Float32Array[] - sampleRate: number - channelCount: number + type: 'reanalyze' + settings: SpectrogramSettings } | { - type: 'finalizeStream' - duration: number + // Re-derive VAD segments from the cached per-frame speech probabilities using new + // thresholds — no re-decode or model re-run, so it's instant. + type: 'resegmentVad' + vadSettings: VadSettings } | { - type: 'reanalyze' - settings: SpectrogramSettings + // Re-run pitch detection with new settings (re-decodes the worker's stored URL). + type: 'reanalyzePitch' + pitchSettings: PitchSettings + pluginSettings?: Record + } + | { + // Compute VAD as a deferred pass (re-decodes) so it doesn't delay spectrogram/waveform. + type: 'analyzeVad' + vadSettings: VadSettings + pluginSettings?: Record } export interface VadSegment { start: number; end: number } +/** Silero VAD segmentation tuning. All operate on the cached per-frame speech probabilities, so + * changing them re-segments instantly (see resegmentVad). */ +export interface VadSettings { + positiveThreshold: number // prob ≥ this opens a speech segment + negativeThreshold: number // prob < this (for redemptionMs) closes it + redemptionMs: number // pause tolerated before a segment ends (shorter → more segments) + minSpeechMs: number // discard speech shorter than this +} + +export const DEFAULT_VAD_SETTINGS: VadSettings = { + positiveThreshold: 0.3, + negativeThreshold: 0.2, + redemptionMs: 250, + minSpeechMs: 150, +} + export interface FrameStat { frameNum: number tSec: number @@ -78,6 +110,34 @@ export interface FrameStat { queueDepth: number } +export type PitchBackend = 'yin' | 'swiftf0' + +/** Derived per-channel pitch track. Computed in the worker; the host may persist the raw arrays to + * `.mumo` (pitch sidecars) and restore them on load, otherwise it's re-derived from audio. Explicit + * per-frame `times` (seconds, frame centers) so both backends' framings compose. `f0 = 0` marks an + * unvoiced frame; `confidence` is the backend's own [0,1] voicing/periodicity score. */ +export interface PitchTrack { + channelIndex: number + backend: PitchBackend + times: Float32Array + f0: Float32Array + confidence: Float32Array +} + +/** Pitch detection tuning. `backend`/`minHz`/`maxHz`/`threshold` re-derive the track (a re-decode, + * like the spectrogram); `confidenceThreshold` is a render-time voicing gate (instant). */ +export interface PitchSettings { + backend: PitchBackend + minHz: number // YIN only (SwiftF0's range is fixed by the model) + maxHz: number // YIN only + threshold: number // YIN CMNDF aperiodicity threshold + confidenceThreshold: number // hide frames below this confidence (voicing gate) +} + +export const DEFAULT_PITCH_SETTINGS: PitchSettings = { + backend: 'swiftf0', minHz: 50, maxHz: 600, threshold: 0.15, confidenceThreshold: 0.5, +} + export type WorkerResponse = | { type: 'decoded'; sampleRate: number; channelCount: number; duration: number } | { type: 'waveform'; channelIndex: number; bins: WaveformBins } @@ -86,5 +146,8 @@ export type WorkerResponse = | { type: 'progress'; done: number; total: number } | { type: 'onsets'; channelIndex: number; timestamps: Float32Array; strengths: Float32Array; bandTimestamps: Float32Array[]; bandStrengths: Float32Array[] } | { type: 'vad'; segments: VadSegment[] } + | { type: 'vadProgress'; done: number; total: number } + | { type: 'pitch'; channelIndex: number; track: PitchTrack } + | { type: 'pitchProgress'; done: number; total: number } | { type: 'error'; message: string } | { type: 'custom'; pluginId: string; data: unknown } diff --git a/packages/media-player/tests/pitch.test.ts b/packages/media-player/tests/pitch.test.ts new file mode 100644 index 0000000..3a46236 --- /dev/null +++ b/packages/media-player/tests/pitch.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from 'vitest' +import { ContinuousResampler16k } from '../src/plugins/signal/pitch.ts' + +// The YIN plugin itself needs the Rust/WASM SampleBuffer (browser/worker only); here we cover the +// pure streaming resampler that feeds it. It must produce a continuous 16 kHz stream whose result +// is identical regardless of how the input is chunked. + +function runChunked(nativeRate: number, totalSamples: number, chunk: number): Float32Array { + const r = new ContinuousResampler16k(nativeRate) + const out: number[] = [] + for (let pos = 0; pos < totalSamples; pos += chunk) { + const n = Math.min(chunk, totalSamples - pos) + const buf = new Float32Array(n) + for (let i = 0; i < n; i++) buf[i] = Math.sin((pos + i) * 0.02) + for (const v of r.push(buf)) out.push(v) + } + return Float32Array.from(out) +} + +describe('ContinuousResampler16k', () => { + it('produces ~ duration * 16000 output samples (48k downsample)', () => { + const seconds = 2 + const out = runChunked(48000, 48000 * seconds, 48000 * seconds) + expect(Math.abs(out.length - 16000 * seconds)).toBeLessThanOrEqual(1) + }) + + it('is chunking-invariant (big vs small vs tiny chunks give identical output)', () => { + const total = 48000 // 1 s + const big = runChunked(48000, total, total) + const small = runChunked(48000, total, 1000) + const tiny = runChunked(48000, total, 333) + expect(small.length).toBe(big.length) + expect(tiny.length).toBe(big.length) + for (let i = 0; i < big.length; i++) { + expect(small[i]).toBeCloseTo(big[i]!, 6) + expect(tiny[i]).toBeCloseTo(big[i]!, 6) + } + }) + + it('preserves a constant signal', () => { + const r = new ContinuousResampler16k(44100) + const out = r.push(new Float32Array(44100).fill(0.5)) + expect(out.length).toBeGreaterThan(0) + for (const v of out) expect(v).toBeCloseTo(0.5, 6) + expect(Math.abs(out.length - 16000)).toBeLessThanOrEqual(2) + }) +}) diff --git a/packages/media-player/tests/praat-pitch.test.ts b/packages/media-player/tests/praat-pitch.test.ts new file mode 100644 index 0000000..cc5272d --- /dev/null +++ b/packages/media-player/tests/praat-pitch.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect } from 'vitest' +import { readFileSync, existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { praatPitchAc, PRAAT_PITCH_DEFAULTS, smoothOctaves } from '../src/plugins/signal/praatPitch.ts' + +// Gold-standard comparison: our JS port of Praat's AC pitch vs Praat 6.6's own output on a real +// 10 s speech segment. Fixtures generated by scripts/praat_wav_reference.praat (committed under +// tests/fixtures/praat/). If the fixtures are absent (not generated), the suite is skipped. + +const dir = resolve(process.cwd(), 'packages/media-player/tests/fixtures/praat') +const wavPath = resolve(dir, 'seg_mono.wav') +const csvPath = resolve(dir, 'seg_pitch.csv') +const haveFixtures = existsSync(wavPath) && existsSync(csvPath) + +function readWavMono(path: string): { sampleRate: number; samples: Float32Array } { + const buf = readFileSync(path) + if (buf.toString('ascii', 0, 4) !== 'RIFF' || buf.toString('ascii', 8, 12) !== 'WAVE') throw new Error('not a WAV') + let pos = 12, sampleRate = 0, bits = 16, channels = 1 + let samples = new Float32Array(0) + while (pos + 8 <= buf.length) { + const id = buf.toString('ascii', pos, pos + 4) + const size = buf.readUInt32LE(pos + 4) + const body = pos + 8 + if (id === 'fmt ') { channels = buf.readUInt16LE(body + 2); sampleRate = buf.readUInt32LE(body + 4); bits = buf.readUInt16LE(body + 14) } + else if (id === 'data') { + const bytesPerSample = bits / 8, frames = Math.floor(size / (bytesPerSample * channels)) + samples = new Float32Array(frames) + for (let f = 0; f < frames; f++) { + // mono fixture (channels === 1); if stereo, average. + let acc = 0 + for (let c = 0; c < channels; c++) { + const o = body + (f * channels + c) * bytesPerSample + acc += bits === 16 ? buf.readInt16LE(o) / 32768 : buf.readInt32LE(o) / 2147483648 + } + samples[f] = acc / channels + } + } + pos = body + size + (size & 1) + } + return { sampleRate, samples } +} + +function readPitchCsv(path: string): { time: number; f0: number }[] { + return readFileSync(path, 'utf8').trim().split('\n').slice(1).map(line => { + const [t, f] = line.split(',') + return { time: parseFloat(t!), f0: parseFloat(f!) } + }) +} + +describe('smoothOctaves', () => { + it('fixes isolated octave errors, keeps steady pitch and unvoiced gaps', () => { + const base = 150 + const f0 = new Float32Array(20).fill(base) + f0[5] = base * 2 // octave-up error + f0[12] = base / 2 // octave-down error + f0[8] = 0 // unvoiced + const out = smoothOctaves(f0) + expect(out[5]).toBeCloseTo(base, 3) + expect(out[12]).toBeCloseTo(base, 3) + expect(out[8]).toBe(0) + for (let i = 0; i < out.length; i++) if (i !== 8) expect(out[i]).toBeCloseTo(base, 3) + }) + it('leaves a smooth glide unchanged', () => { + const f0 = new Float32Array(20) + for (let i = 0; i < 20; i++) f0[i] = 120 + i * 3 // 120→177, gradual + const out = smoothOctaves(f0) + for (let i = 0; i < 20; i++) expect(out[i]).toBeCloseTo(f0[i]!, 3) + }) +}) + +describe('praatPitchAc synthetic', () => { + it('tracks a pure 150 Hz sine and a 120→180 Hz glide', () => { + const sr = 48000, n = sr * 1 + const sine = new Float32Array(n) + for (let i = 0; i < n; i++) sine[i] = 0.5 * Math.sin(2 * Math.PI * 150 * (i + 0.5) / sr) + const rs = praatPitchAc(sine, sr, PRAAT_PITCH_DEFAULTS) + const voiced = [...rs.f0].filter(f => f > 0) + expect(voiced.length).toBeGreaterThan(rs.f0.length * 0.9) + for (const f of voiced) expect(Math.abs(1200 * Math.log2(f / 150))).toBeLessThan(5) // < 5 cents + + const glide = new Float32Array(n) + for (let i = 0; i < n; i++) { const t = (i + 0.5) / sr; glide[i] = 0.5 * Math.sin(2 * Math.PI * (120 * t + 30 * t * t)) } + const rg = praatPitchAc(glide, sr, PRAAT_PITCH_DEFAULTS) + // Instantaneous freq 120→180: each voiced frame's f0 should be within [110, 190]. + for (let i = 0; i < rg.f0.length; i++) if (rg.f0[i]! > 0) { expect(rg.f0[i]!).toBeGreaterThan(110); expect(rg.f0[i]!).toBeLessThan(190) } + }) +}) + +describe.skipIf(!haveFixtures)('praatPitchAc vs Praat 6.6 (gold standard)', () => { + const { sampleRate, samples } = haveFixtures ? readWavMono(wavPath) : { sampleRate: 0, samples: new Float32Array(0) } + const ref = haveFixtures ? readPitchCsv(csvPath) : [] + + it('matches Praat: frame grid, voicing, and cents error', () => { + const out = praatPitchAc(samples, sampleRate, PRAAT_PITCH_DEFAULTS) + expect(out.f0.length).toBe(ref.length) + + let bothVoiced = 0, voicingAgree = 0, octaveErrors = 0 + const centsErrors: number[] = [] + for (let i = 0; i < ref.length; i++) { + const rf = ref[i]!.f0, of = out.f0[i]! + const rv = rf > 0, ov = of > 0 + if (rv === ov) voicingAgree++ + if (rv && ov) { + bothVoiced++ + const cents = 1200 * Math.log2(of / rf) + centsErrors.push(Math.abs(cents)) + if (Math.abs(cents) > 600) octaveErrors++ // >½ octave off ⇒ likely octave error + } + } + centsErrors.sort((a, b) => a - b) + const median = centsErrors[Math.floor(centsErrors.length / 2)] ?? 0 + const p90 = centsErrors[Math.floor(centsErrors.length * 0.9)] ?? 0 + const voicingPct = (voicingAgree / ref.length) * 100 + const octPct = bothVoiced ? (octaveErrors / bothVoiced) * 100 : 0 + + console.log(`[praat-pitch] frames=${ref.length} bothVoiced=${bothVoiced} voicingAgree=${voicingPct.toFixed(1)}% ` + + `medianCents=${median.toFixed(1)} p90Cents=${p90.toFixed(1)} octaveErr=${octPct.toFixed(1)}%`) + + // Our JS port matches Praat 6.6 to well under a cent (only interpolation differs: parabolic vs + // Praat's sinc), with identical voicing and no octave errors. + expect(voicingPct).toBeGreaterThan(99) + expect(median).toBeLessThan(3) + expect(p90).toBeLessThan(8) + expect(octPct).toBeLessThan(1) + }) +}) diff --git a/packages/media-player/tests/silero-vad.test.ts b/packages/media-player/tests/silero-vad.test.ts new file mode 100644 index 0000000..e5a9650 --- /dev/null +++ b/packages/media-player/tests/silero-vad.test.ts @@ -0,0 +1,125 @@ +import { describe, it, expect } from 'vitest' +import { Resampler16k, segmentProbs } from '../src/plugins/signal/sileroVad.ts' +import { mergeVadSegments } from '../src/plugins/signal/vad.ts' +import { DEFAULT_VAD_SETTINGS } from '../src/types.ts' + +// The full Silero path (ONNX + model fetch) can only run in the Electron/browser worker, so these +// tests cover the deterministic pieces: the streaming resampler and per-channel segment merging. + +describe('Resampler16k streaming resampler', () => { + function frameCount(nativeRate: number, seconds: number, chunk: number): { frames: number; total: number } { + const r = new Resampler16k(nativeRate, 1536) + const total = Math.round(nativeRate * seconds) + let frames = 0 + for (let pos = 0; pos < total; pos += chunk) { + const n = Math.min(chunk, total - pos) + const buf = new Float32Array(n) + for (let i = 0; i < n; i++) buf[i] = Math.sin((pos + i) * 0.01) + frames += r.push(buf).length + } + return { frames, total } + } + + it('produces ~duration*16000/1536 frames regardless of chunking (48k downsample)', () => { + const seconds = 5 + const expected = Math.floor((seconds * 16000) / 1536) + // Feeding in one big chunk vs many small chunks must yield the same frame count (streaming + // must not lose or duplicate samples at chunk boundaries). + const big = frameCount(48000, seconds, 48000 * seconds) + const small = frameCount(48000, seconds, 511) + const tiny = frameCount(48000, seconds, 97) + expect(big.frames).toBe(small.frames) + expect(big.frames).toBe(tiny.frames) + expect(Math.abs(big.frames - expected)).toBeLessThanOrEqual(1) + }) + + it('handles 44.1k and 16k (no-op) rates', () => { + const secs = 3 + const at441 = frameCount(44100, secs, 1000).frames + const at16 = frameCount(16000, secs, 1000).frames + expect(Math.abs(at441 - Math.floor((secs * 16000) / 1536))).toBeLessThanOrEqual(1) + expect(Math.abs(at16 - Math.floor((secs * 16000) / 1536))).toBeLessThanOrEqual(1) + }) + + it('emits exactly frameSize-length frames', () => { + const r = new Resampler16k(48000, 1536) + const frames = r.push(new Float32Array(48000)) // 1s + for (const f of frames) expect(f.length).toBe(1536) + expect(frames.length).toBeGreaterThan(0) + }) + + it('preserves a constant signal through interpolation', () => { + const r = new Resampler16k(48000, 1536) + const input = new Float32Array(48000).fill(0.5) + const frames = r.push(input) + for (const f of frames) for (const v of f) expect(v).toBeCloseTo(0.5, 6) + }) +}) + +describe('segmentProbs (settings-driven re-segmentation)', () => { + // 5 speech frames, 3 low frames, 5 speech frames. Frame = 96 ms. + const probs = [0.9, 0.9, 0.9, 0.9, 0.9, 0.0, 0.0, 0.0, 0.9, 0.9, 0.9, 0.9, 0.9] + + it('short redemption splits on the gap; long redemption merges through it', () => { + const split = segmentProbs(probs, { ...DEFAULT_VAD_SETTINGS, redemptionMs: 250, minSpeechMs: 150 }) + expect(split.length).toBe(2) + + const merged = segmentProbs(probs, { ...DEFAULT_VAD_SETTINGS, redemptionMs: 1400, minSpeechMs: 150 }) + expect(merged.length).toBe(1) + }) + + it('minSpeechMs discards bursts shorter than the threshold', () => { + const brief = [0.9, 0.0, 0.0, 0.0, 0.0] // one speech frame (~96 ms) + expect(segmentProbs(brief, { ...DEFAULT_VAD_SETTINGS, redemptionMs: 250, minSpeechMs: 250 })).toEqual([]) + expect(segmentProbs(brief, { ...DEFAULT_VAD_SETTINGS, redemptionMs: 250, minSpeechMs: 90 }).length).toBe(1) + }) + + it('threshold controls whether mid-level probability counts as speech', () => { + const mid = [0.45, 0.45, 0.45, 0.45, 0.0, 0.0, 0.0, 0.0] + expect(segmentProbs(mid, { ...DEFAULT_VAD_SETTINGS, positiveThreshold: 0.3 }).length).toBe(1) + expect(segmentProbs(mid, { ...DEFAULT_VAD_SETTINGS, positiveThreshold: 0.5 })).toEqual([]) + }) + + it('returns empty for all-silence', () => { + expect(segmentProbs([0, 0, 0, 0], DEFAULT_VAD_SETTINGS)).toEqual([]) + }) + + it('ends at the last speech frame, not padded by the redemption window', () => { + // 5 speech frames (0..4), then 10 silent frames. Frame = 96 ms. + const probs = [0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + const [seg] = segmentProbs(probs, { ...DEFAULT_VAD_SETTINGS, redemptionMs: 250, minSpeechMs: 150 }) + expect(seg).toBeDefined() + // End should be frame 5 (right after last speech frame 4) = 5 * 96 ms, NOT the redemption + // expiry a few frames later. + expect(seg!.end).toBeCloseTo((5 * 96) / 1000, 6) + expect(seg!.start).toBeCloseTo(0, 6) + }) +}) + +describe('mergeVadSegments', () => { + it('coalesces overlapping and near-touching segments across channels', () => { + const merged = mergeVadSegments([ + [{ start: 0.0, end: 1.0 }, { start: 3.0, end: 4.0 }], + [{ start: 0.9, end: 1.5 }, { start: 3.02, end: 3.5 }], // 3.02 within 0.05 of 3.0..4.0 + ]) + expect(merged).toEqual([ + { start: 0.0, end: 1.5 }, + { start: 3.0, end: 4.0 }, + ]) + }) + + it('keeps well-separated segments distinct and sorted', () => { + const merged = mergeVadSegments([ + [{ start: 5.0, end: 6.0 }], + [{ start: 0.0, end: 1.0 }], + ]) + expect(merged).toEqual([ + { start: 0.0, end: 1.0 }, + { start: 5.0, end: 6.0 }, + ]) + }) + + it('returns empty for no speech', () => { + expect(mergeVadSegments([[], []])).toEqual([]) + }) +}) diff --git a/packages/media-player/tests/streaming-signal.test.ts b/packages/media-player/tests/streaming-signal.test.ts new file mode 100644 index 0000000..2046d2f --- /dev/null +++ b/packages/media-player/tests/streaming-signal.test.ts @@ -0,0 +1,176 @@ +import { describe, it, expect } from 'vitest' +import { spectrogramPlugin, specFrameParams } from '../src/plugins/signal/spectrogram.ts' +import { waveformPlugin } from '../src/plugins/signal/waveform.ts' +import { SegmentProducer } from '../src/segmenter.ts' +import { DEFAULT_SPEC_SETTINGS } from '../src/types.ts' +import type { SignalPlugin } from '../src/plugins/signal/SignalPlugin.ts' +import type { WorkerResponse } from '../src/types.ts' + +// The worker decodes the file in ~90 s segments so full-file PCM is never resident. These tests +// drive the streaming plugins through SegmentProducer at different segment sizes and assert the +// segmented output is byte-identical to the whole-file (single-segment) output. WASM never loads +// in node, so this exercises the JS fallback path. + +const TILE_FRAMES = 2048 + +interface Collected { + tiles: Map // `${ch}:${tileIndex}` -> rawDb + tileDims: Map + tileTimeEnd: Map + overview: Map + overviewMeta: Map + waveform: Map + lastProgress: { done: number; total: number } | null +} + +/** Build a deterministic, per-channel-distinct signal with a slow amplitude envelope. */ +function makeSignal(channelCount: number, totalSamples: number, sampleRate: number): Float32Array[] { + return Array.from({ length: channelCount }, (_, ch) => { + const s = new Float32Array(totalSamples) + const f0 = 220 + ch * 110 + for (let i = 0; i < totalSamples; i++) { + const t = i / sampleRate + const env = 0.5 + 0.5 * Math.sin(2 * Math.PI * 0.7 * t + ch) + s[i] = env * (0.6 * Math.sin(2 * Math.PI * f0 * t) + 0.3 * Math.sin(2 * Math.PI * f0 * 2.5 * t + 1)) + } + return s + }) +} + +/** Run a plugin over the signal, cutting into segments of `segFrames` frames, collecting posts. */ +async function run( + plugin: SignalPlugin, signal: Float32Array[], sampleRate: number, + totalFrames: number, hop: number, windowSize: number, segFrames: number, durationSec: number, +): Promise { + const channelCount = signal.length + const out: Collected = { tiles: new Map(), tileDims: new Map(), tileTimeEnd: new Map(), overview: new Map(), overviewMeta: new Map(), waveform: new Map(), lastProgress: null } + const post = (msg: WorkerResponse): void => { + if (msg.type === 'spectrogramTile') { + const key = `${msg.channelIndex}:${msg.tile.tileIndex}` + out.tiles.set(key, msg.tile.rawDb as Uint8Array) + out.tileDims.set(key, [msg.tile.width, msg.tile.height]) + out.tileTimeEnd.set(key, msg.tile.timeEnd) + } else if (msg.type === 'spectrogramOverview') { + out.overview.set(msg.channelIndex, msg.tile.rawDb as Uint8Array) + out.overviewMeta.set(msg.channelIndex, { width: msg.tile.width, timeEnd: msg.tile.timeEnd }) + } else if (msg.type === 'waveform') { + out.waveform.set(msg.channelIndex, { peakPos: msg.bins.peakPos, peakNeg: msg.bins.peakNeg, rms: msg.bins.rms, binCount: msg.bins.binCount }) + } else if (msg.type === 'progress') { + out.lastProgress = { done: msg.done, total: msg.total } + } + } + + const runInst = plugin.createRun( + { sampleRate, channelCount, durationSec, settings: DEFAULT_SPEC_SETTINGS, pluginSettings: {}, trigger: 'analyze' }, post, + ) + const producer = new SegmentProducer(channelCount, hop, windowSize, segFrames, totalFrames) + + // Feed decoded PCM in small irregular chunks, draining ready segments as we go. + let pos = 0 + const chunkSizes = [1000, 1500, 777, 2048, 333] + let ci = 0 + const total = signal[0]!.length + const drain = async (final: boolean): Promise => { + let seg + while ((seg = producer.tryCut(final)) !== null) await runInst.pushSegment(seg) + } + while (pos < total) { + const n = Math.min(chunkSizes[ci++ % chunkSizes.length]!, total - pos) + const perCh = signal.map(ch => ch.subarray(pos, pos + n)) + producer.add(perCh.map(a => Float32Array.from(a)), n) + pos += n + await drain(false) + } + await drain(true) + await runInst.finish() + return out +} + +function expectUint8Equal(a: Uint8Array, b: Uint8Array, label: string): void { + expect(a.length, `${label} length`).toBe(b.length) + // Fast path: find first mismatch for a useful message. + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) { expect.fail(`${label} differs at ${i}: ${a[i]} vs ${b[i]}`) } + } +} + +function expectFloatEqual(a: Float32Array, b: Float32Array, label: string): void { + expect(a.length, `${label} length`).toBe(b.length) + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) { expect.fail(`${label} differs at ${i}: ${a[i]} vs ${b[i]}`) } + } +} + +describe('streaming signal analysis is segmentation-invariant', () => { + const sampleRate = 16000 + const channelCount = 2 + const { hop, windowSize } = specFrameParams(DEFAULT_SPEC_SETTINGS, sampleRate) + + // Size the signal to span several tiles so multi-segment cuts land on real tile boundaries. + const targetFrames = TILE_FRAMES * 3 + 137 // not a tile multiple -> exercises a short final tile + const totalSamples = (targetFrames - 1) * hop + windowSize + 91 // +91: partial trailing frame + const durationSec = totalSamples / sampleRate + const totalFrames = Math.max(1, Math.floor((totalSamples - windowSize) / hop) + 1) + + const signal = makeSignal(channelCount, totalSamples, sampleRate) + + it('sanity: spans multiple tiles', () => { + expect(totalFrames).toBeGreaterThan(TILE_FRAMES * 3) + }) + + it('spectrogram tiles + overview match across segment sizes', { timeout: 60000 }, async () => { + const whole = await run(spectrogramPlugin, signal, sampleRate, totalFrames, hop, windowSize, totalFrames, durationSec) + const oneTile = await run(spectrogramPlugin, signal, sampleRate, totalFrames, hop, windowSize, TILE_FRAMES, durationSec) + const twoTile = await run(spectrogramPlugin, signal, sampleRate, totalFrames, hop, windowSize, TILE_FRAMES * 2, durationSec) + + for (const [label, segmented] of [['1-tile', oneTile], ['2-tile', twoTile]] as const) { + expect(segmented.tiles.size, `${label} tile count`).toBe(whole.tiles.size) + for (const [key, rawDb] of whole.tiles) { + const other = segmented.tiles.get(key) + expect(other, `${label} missing tile ${key}`).toBeDefined() + expect(segmented.tileDims.get(key), `${label} dims ${key}`).toEqual(whole.tileDims.get(key)) + expectUint8Equal(other!, rawDb, `${label} tile ${key}`) + } + for (const [ch, ov] of whole.overview) { + expectUint8Equal(segmented.overview.get(ch)!, ov, `${label} overview ch${ch}`) + } + } + }) + + it('overview extent + progress reconcile to actual frames when duration is over-reported', { timeout: 60000 }, async () => { + // Simulate a container that reports 40% more audio than it actually decodes (the condition + // that previously stretched the overview and pinned progress below 100%). + const inflatedDuration = durationSec * 1.4 + const inflatedSamples = Math.round(inflatedDuration * sampleRate) + const inflatedFrames = Math.max(1, Math.floor((inflatedSamples - windowSize) / hop) + 1) + const actualFrames = totalFrames // frames the (real) signal actually yields + + const res = await run(spectrogramPlugin, signal, sampleRate, inflatedFrames, hop, windowSize, TILE_FRAMES * 2, inflatedDuration) + + // Overview ends at the real content, not the inflated duration. + const expectedEnd = actualFrames * (hop / sampleRate) + for (const [, meta] of res.overviewMeta) { + expect(meta.timeEnd).toBeCloseTo(expectedEnd, 3) + expect(meta.timeEnd).toBeLessThan(inflatedDuration - 0.5) + expect(meta.width).toBeLessThanOrEqual(Math.ceil(actualFrames / Math.ceil(inflatedFrames / Math.min(inflatedFrames, 4096)))) + } + // No tile extends past the real content end. + for (const [, te] of res.tileTimeEnd) expect(te).toBeLessThanOrEqual(expectedEnd + 1e-6) + // Progress bar reaches 100% despite the over-estimate. + expect(res.lastProgress).not.toBeNull() + expect(res.lastProgress!.done).toBe(res.lastProgress!.total) + }) + + it('waveform bins match across segment sizes', async () => { + const whole = await run(waveformPlugin, signal, sampleRate, totalFrames, hop, windowSize, totalFrames, durationSec) + const small = await run(waveformPlugin, signal, sampleRate, totalFrames, hop, windowSize, TILE_FRAMES, durationSec) + + for (const [ch, w] of whole.waveform) { + const s = small.waveform.get(ch)! + expect(s.binCount, `waveform binCount ch${ch}`).toBe(w.binCount) + expectFloatEqual(s.peakPos, w.peakPos, `waveform peakPos ch${ch}`) + expectFloatEqual(s.peakNeg, w.peakNeg, `waveform peakNeg ch${ch}`) + expectFloatEqual(s.rms, w.rms, `waveform rms ch${ch}`) + } + }) +}) diff --git a/packages/mumo/src/App.svelte b/packages/mumo/src/App.svelte index 7650eb1..9e3eae0 100644 --- a/packages/mumo/src/App.svelte +++ b/packages/mumo/src/App.svelte @@ -2,7 +2,7 @@ import { onMount, untrack } from 'svelte' import { SvelteMap, SvelteSet } from 'svelte/reactivity' import * as Y from 'yjs' - import { TranscriptEditor, TranscriptOverlay, OverlapOverlayPlugin, BlockHighlightOverlayPlugin, PatternOverlayPlugin, initYXmlFragment, ySyncPluginKey, setAllGlosses, setGlossesVisible, resolveTokenRanges, setUttTiersVisible } from '@mumo/editor' + import { TranscriptEditor, TranscriptOverlay, OverlapOverlayPlugin, BlockHighlightOverlayPlugin, PatternOverlayPlugin, initYXmlFragment, ySyncPluginKey, setAllGlosses, setGlossesVisible, resolveTokenRanges, setUttTiersVisible, redrawAllProsody } from '@mumo/editor' import type { TokenRef, GlossEntry, FormattingState, PatternOverlayEntry } from '@mumo/editor' import { Timeline } from '@mumo/timeline' import type { SnapPlugin, SnapMode, CommitEntry } from '@mumo/timeline' @@ -14,7 +14,7 @@ import { MediaResolver } from './media-resolver.js' import appIconUrl from './assets/mumo.svg' import magnetIconUrl from './assets/magnet.svg' - import type { EAFDocument, EAFMediaDescriptor, MumoImageInput, MumoSpectrogramInput, MumoTrackBufferInput } from '@mumo/serialization' + import type { EAFDocument, EAFMediaDescriptor, MumoImageInput, MumoSpectrogramInput, MumoTrackBufferInput, MumoPitchInput, PitchConfigMeta, PitchSettingsMeta } from '@mumo/serialization' import { FileController } from './fileController.js' import type { ImportResult } from './formats.js' import { WebPlatformIO, guessMime } from './platform.js' @@ -66,8 +66,8 @@ import EditTierDlg from './dialogs/EditTierDlg.svelte' import UttTiersDlg from './dialogs/UttTiersDlg.svelte' import type { SlotFillMode } from './patternTypes.js' - import type { MediaState, SpectrogramSettings, VadSegment, WaveformBins } from '@mumo/media-player' - import { SPEC_PRESETS, DEFAULT_SPEC_SETTINGS, MultiMediaPlayer, VideoTileLayout, LinkedMediaDlg, computeEnergyVad } from '@mumo/media-player' + import type { MediaState, SpectrogramSettings, VadSegment, VadSettings, PitchSettings, WaveformBins } from '@mumo/media-player' + import { SPEC_PRESETS, DEFAULT_SPEC_SETTINGS, DEFAULT_VAD_SETTINGS, DEFAULT_PITCH_SETTINGS, MultiMediaPlayer, VideoTileLayout, LinkedMediaDlg, computeEnergyVad, smoothOctaves } from '@mumo/media-player' import type { MediaPlayer } from '@mumo/media-player' import type { SignalChannel, TickMark, TierIntervalOverlay, ArcItem, MotionCurve } from '@mumo/timeline' import './css/base.css' @@ -658,6 +658,184 @@ let mediaSignals = $state([]) let hiddenSignalIds = $state>(new Set()) + // Pitch is drawn over the waveform (not its own lane) and is off by default, toggled from the + // timeline gear menu. Contours are held here keyed by waveform-signal id and attached to the + // matching spectrogram channel while that channel's id is in `pitchChannels`. + const pitchChannels = new SvelteSet() // spectrogram-signal ids with the pitch overlay on + let pitchComputing = $state(false) // a pitch pass is running (drives the progress bar) + let pitchComputed = $state(false) // pitch has been computed for the current primary file + let pitchProgress = $state<{ done: number; total: number } | null>(null) + let _lastPitchFile: string | undefined // primary filename analyzer state is scoped to + let pitchSettings = $state({ ...DEFAULT_PITCH_SETTINGS }) + // VAD (Silero) runs as a deferred pass after the spectrogram/waveform appear; enable-able. + let vadEnabled = $state(true) + let vadComputed = $state(false) + let vadComputing = $state(false) + let vadProgress = $state<{ done: number; total: number } | null>(null) + // Raw per-channel pitch tracks keyed by waveform-signal id (kept so the confidence gate can be + // re-applied instantly without re-detecting). + // eslint-disable-next-line svelte/prefer-svelte-reactivity -- imperative cache; reactivity flows through mediaSignals via _syncPitchOverlay + const pitchTracks = new Map() + // Per-channel pitch-settings overrides (empty today; the UI edits only the global `pitchSettings`, + // but the format/round-trip already carry per-channel overrides so a noisier channel can be tuned + // separately later). Keyed by `${mediaKey}ch${idx}`; falls back to the global `pitchSettings`. + // eslint-disable-next-line svelte/prefer-svelte-reactivity -- imperative map; not rendered directly + const pitchChannelSettings = new Map() + // Pitch tracks loaded from a .mumo, awaiting their channel's waveform signal so they can be re-keyed + // by the (new session) playerId. Keyed by `${mediaKey}ch${idx}`. Consumed in onWaveform. + // eslint-disable-next-line svelte/prefer-svelte-reactivity -- imperative map; drained on load + const _pendingLoadedPitch = new Map() + + // Stable identity for a media item across save/reload: the file path, else its filename. NOT a + // content hash — that isn't reliably available (desktop reload uses an empty File over a path) — + // and NOT the player id or array index (per-session / decode-order dependent). + function _mediaKey(player: { track?: { path?: string | null } | null; state?: { filename?: string } | null } | undefined): string { + return player?.track?.path ?? player?.state?.filename ?? '' + } + const _pitchKey = (mediaKey: string, ch: number): string => `${mediaKey}ch${ch}` + + function resolvePitchSettings(mediaKey: string, ch: number): PitchSettings { + return pitchChannelSettings.get(_pitchKey(mediaKey, ch))?.settings ?? pitchSettings + } + + function _metaToPitchSettings(m: PitchSettingsMeta): PitchSettings { + return { + backend: m.backend === 'yin' ? 'yin' : 'swiftf0', + minHz: m.minHz, maxHz: m.maxHz, threshold: m.threshold, confidenceThreshold: m.confidenceThreshold, + } + } + + // Pitch-generation metadata for `.mmeaf` (global defaults + any per-channel overrides). + function buildPitchConfig(): PitchConfigMeta { + const toMeta = (s: PitchSettings): PitchSettingsMeta => ({ + backend: s.backend, minHz: s.minHz, maxHz: s.maxHz, threshold: s.threshold, confidenceThreshold: s.confidenceThreshold, + }) + const channels: NonNullable = [] + for (const v of pitchChannelSettings.values()) { + channels.push({ mediaKey: v.mediaKey, channelIndex: v.channelIndex, settings: toMeta(v.settings) }) + } + return { defaults: toMeta(pitchSettings), ...(channels.length ? { channels } : {}) } + } + + function _buildPitchOverlay(track: { times: Float32Array; f0: Float32Array; confidence: Float32Array }): { samples: Array<[number, number]>; yMin: number; yMax: number } | null { + const { times, confidence } = track + const thr = pitchSettings.confidenceThreshold + const loHz = pitchSettings.minHz, hiHz = pitchSettings.maxHz + // Octave-jump correction (post-processing) then linear frequency. Filter by confidence AND the + // display frequency range; out-of-range / low-confidence frames become NaN gaps. + const f0 = smoothOctaves(track.f0) + const samples: Array<[number, number]> = new Array(f0.length) + const vals: number[] = [] + for (let f = 0; f < f0.length; f++) { + const hz = f0[f]! + const voiced = hz >= loHz && hz <= hiHz && confidence[f]! >= thr + if (voiced) { samples[f] = [times[f]!, hz]; vals.push(hz) } + else samples[f] = [times[f]!, NaN] + } + if (vals.length === 0) return null // nothing voiced/in-range + // Scale to the robust bulk of the pitch (2nd–98th percentile) so residual octave errors don't + // compress the range. Floor the span so near-constant pitch isn't magnified into noise. + vals.sort((a, b) => a - b) + const pct = (q: number) => vals[Math.min(vals.length - 1, Math.floor(q * (vals.length - 1)))]! + let yMin = pct(0.02), yMax = pct(0.98) + const MIN_SPAN = 30 // Hz + if (yMax - yMin < MIN_SPAN) { const mid = (yMin + yMax) / 2; yMin = mid - MIN_SPAN / 2; yMax = mid + MIN_SPAN / 2 } + const pad = (yMax - yMin) * 0.1 + return { samples, yMin: Math.max(0, yMin - pad), yMax: yMax + pad } + } + + function _syncPitchOverlay() { + _intonationCache.clear() // pitch filter/detect changed → intonation contours must rebuild + mediaSignals = mediaSignals.map(s => { + if (s.kind !== 'waveform') return s + const track = pitchChannels.has(s.id) ? pitchTracks.get(s.id) : undefined + const overlay = track ? _buildPitchOverlay(track) : null + if (overlay) return { ...s, pitch: overlay } + if (!s.pitch) return s + const rest = { ...s } + delete rest.pitch // omit (not undefined) — exactOptionalPropertyTypes + return rest + }) + _flushSignals() + // Pitch data changed (compute, load-inject, or filter tweak) → refresh transcript intonation + // contours, which read pitch via getIntonation but don't observe it. rAF-debounced per nodeview. + redrawAllProsody() + } + + // Intonation contour data for a transcript block: the f0 samples of `channel` over [t0, t1] plus + // the channel's robust y-scale. Reuses the pitch-overlay build; cached by channel-key + track + // identity (a new track object on recompute invalidates automatically; cleared on filter change). + type IntonationOverlay = { samples: Array<[number, number]>; yMin: number; yMax: number } + // eslint-disable-next-line svelte/prefer-svelte-reactivity -- imperative cache; keyed by track identity, cleared in _syncPitchOverlay + const _intonationCache = new Map() + // Audio channels available for intonation, derived from the waveform signals (id `…:waveform:chN`). + function getAudioChannels(): Array<{ index: number; label: string }> { + // eslint-disable-next-line svelte/prefer-svelte-reactivity -- local dedup map, not reactive state + const seen = new Map() + for (const s of mediaSignals) { + if (s.kind !== 'waveform') continue + const m = /:waveform:ch(\d+)$/.exec(s.id) + if (!m) continue + const idx = parseInt(m[1]!, 10) + if (!seen.has(idx)) seen.set(idx, s.label || `Ch ${idx}`) + } + return [...seen.entries()].map(([index, label]) => ({ index, label })).sort((a, b) => a.index - b.index) + } + + // A participant's default audio channel (set in the participants dialog), or null if unset. + function getParticipantChannel(participant: string): number | null { + const p = participants.find(pp => pp.label === participant) + return p?.channel ?? null + } + + function getIntonation(channel: number, t0: number, t1: number): IntonationOverlay | null { + let key: string | undefined + let track: { times: Float32Array; f0: Float32Array; confidence: Float32Array } | undefined + for (const [k, tr] of pitchTracks) { + if (k.endsWith(`:waveform:ch${channel}`)) { key = k; track = tr; break } + } + if (!key || !track) return null + let entry = _intonationCache.get(key) + if (!entry || entry.track !== track) { + entry = { track, overlay: _buildPitchOverlay(track) } + _intonationCache.set(key, entry) + } + const ov = entry.overlay + if (!ov) return null + const samples = ov.samples.filter(([t]) => t >= t0 && t <= t1) + if (samples.length === 0) return null + return { samples, yMin: ov.yMin, yMax: ov.yMax } + } + + // Pitch is computed lazily: only when the overlay is on and it isn't already computed/running, + // and only once the audio has been analyzed (waveform present). Kicks off a pitch-only re-decode. + function _maybeComputePitch() { + if (pitchChannels.size === 0 || pitchComputed || pitchComputing) return + if (!mediaSignals.some(s => s.kind === 'waveform')) return + pitchComputing = true + pitchProgress = { done: 0, total: 0 } + multiPlayer.computePitch() + } + + // VAD runs as a deferred pass once audio is analyzed — only if enabled and not already done/running. + function _maybeComputeVad() { + if (!vadEnabled || vadComputed || vadComputing) return + if (!mediaSignals.some(s => s.kind === 'waveform')) return + vadComputing = true + vadProgress = { done: 0, total: 0 } + multiPlayer.computeVad() + } + + // Reset analyzer state when the primary file changes (drop stale tracks; recompute lazily). + function _resetPitchForFile(filename: string | undefined) { + if (filename === _lastPitchFile) return + _lastPitchFile = filename + const pid = multiPlayer.primary?.id + for (const k of [...pitchTracks.keys()]) if (!pid || k.startsWith(pid + ':')) pitchTracks.delete(k) + pitchComputed = false; pitchComputing = false; pitchProgress = null + vadComputed = false; vadComputing = false; vadProgress = null + } + function _afterStoreChange() { _recomputeWarnings() if (!_timelinePushPending) { @@ -1166,6 +1344,50 @@ return () => document.removeEventListener('mumo:menu-action', handleMenuAction) }) + // Make modal dialogs draggable by their title/header bar. One delegated handler + // covers every registered dialog type via a (handle → panel) selector pair. Since the + // dialogs are conditionally rendered ({#if …open}), each reopen remounts a fresh + // element with no inline styles, so the position resets on its own. + onMount(() => { + // handle = the grabbable title bar; panel = the element that actually moves. + const DRAG_TARGETS: { handle: string; panel: string }[] = [ + { handle: '.dlg-header, .dlg > h3', panel: '.dlg' }, // standard .dlg dialogs + { handle: '.lmd-header', panel: '.lmd-panel' }, // linked-media dialog + ] + function onDlgMouseDown(e: MouseEvent) { + if (e.button !== 0) return + const target = e.target as HTMLElement | null + if (!target) return + // Grab only the title bar; ignore interactive controls living in the header. + if (target.closest('button, input, select, textarea, a')) return + let panel: HTMLElement | null = null + for (const { handle, panel: panelSel } of DRAG_TARGETS) { + if (target.closest(handle)) { panel = target.closest(panelSel) as HTMLElement | null; break } + } + if (!panel) return + e.preventDefault() + const rect = panel.getBoundingClientRect() + const startX = e.clientX, startY = e.clientY + const origX = rect.left, origY = rect.top + panel.style.left = `${origX}px` + panel.style.top = `${origY}px` + panel.style.transform = 'none' + panel.style.margin = '0' + const onMove = (ev: MouseEvent) => { + panel.style.left = `${origX + (ev.clientX - startX)}px` + panel.style.top = `${origY + (ev.clientY - startY)}px` + } + const onUp = () => { + window.removeEventListener('mousemove', onMove) + window.removeEventListener('mouseup', onUp) + } + window.addEventListener('mousemove', onMove) + window.addEventListener('mouseup', onUp) + } + document.addEventListener('mousedown', onDlgMouseDown) + return () => document.removeEventListener('mousedown', onDlgMouseDown) + }) + // Library imperative API /** Replace the current document (library API). */ @@ -1244,8 +1466,26 @@ let primaryFrameRate = $state(30) // mediaSignals / hiddenSignalIds are declared above _afterStoreChange (TDZ). let spectrogramSettings = $state({ ...DEFAULT_SPEC_SETTINGS }) + let vadSettings = $state({ ...DEFAULT_VAD_SETTINGS }) let spectrogramProgress = $state<{ done: number; total: number } | null>(null) let specModalOpen = $state(false) + let specModalTab = $state<'spectrogram' | 'vad' | 'pitch'>('spectrogram') + // Drag position for the Audio-processing modal; null = default anchored spot (reset on each open). + let specModalPos = $state<{ x: number; y: number } | null>(null) + function startSpecDrag(e: MouseEvent) { + if ((e.target as HTMLElement).closest('.spec-modal-close')) return // let the close button work + e.preventDefault() + const modalEl = (e.currentTarget as HTMLElement).closest('.spec-modal') as HTMLElement | null + if (!modalEl) return + const rect = modalEl.getBoundingClientRect() + const startX = e.clientX, startY = e.clientY + const origX = rect.left, origY = rect.top + specModalPos = { x: origX, y: origY } + const onMove = (ev: MouseEvent) => { specModalPos = { x: origX + (ev.clientX - startX), y: origY + (ev.clientY - startY) } } + const onUp = () => { window.removeEventListener('mousemove', onMove); window.removeEventListener('mouseup', onUp) } + window.addEventListener('mousemove', onMove) + window.addEventListener('mouseup', onUp) + } let hiddenLaneIds = $state>(new Set()) let linkedMediaOpen = $state(false) let linkedPlayers = $state([]) @@ -1322,7 +1562,11 @@ } function _flushSignals() { - timelineRef?.setSignals(mediaSignals.filter(s => !hiddenSignalIds.has(s.id))) + // A hidden waveform whose pitch is on is still shown — but bars suppressed (pitch only). + const out = mediaSignals + .filter(s => !hiddenSignalIds.has(s.id) || (s.kind === 'waveform' && pitchChannels.has(s.id))) + .map(s => (s.kind === 'waveform' && hiddenSignalIds.has(s.id) && pitchChannels.has(s.id)) ? { ...s, pitchOnly: true } : s) + timelineRef?.setSignals(out) } const customSignalHandlers = new Map void>() @@ -1333,6 +1577,7 @@ // Clear only the primary player's signals; secondary signals survive primary reload const pid = multiPlayer.primary?.id if (pid) mediaSignals = mediaSignals.filter(s => !s.id.startsWith(pid + ':')) + _resetPitchForFile(state?.filename) // drop stale pitch on file change (recomputes lazily) mediaState = state if (state) { timelineRef?.setMediaDuration(state.duration) @@ -1371,6 +1616,22 @@ { id, label, kind: 'waveform' as const, waveformBins: bins, height: 20, timeOffset }, ].sort(sortSignals) _flushSignals() + // Restore persisted pitch for this channel (from a loaded .mumo) instead of recomputing. + // Keyed by media identity (path/filename), so it survives the player's new session id. + const mediaKey = _mediaKey(player) + if (mediaKey) { + const pending = _pendingLoadedPitch.get(_pitchKey(mediaKey, ch)) + if (pending) { + pitchTracks.set(id, { times: pending.times, f0: pending.f0, confidence: pending.confidence }) + if (pending.enabled) pitchChannels.add(id) // restore the waveform overlay toggle + _pendingLoadedPitch.delete(_pitchKey(mediaKey, ch)) + pitchComputed = true // suppresses _maybeComputePitch below (no redundant re-detect) + } + } + // Audio analyzed → run the deferred analyzers (VAD if enabled, pitch for any enabled channel). + _maybeComputeVad() + _maybeComputePitch() + _syncPitchOverlay() }, onSpectrogramOverview(playerId, ch, tile) { const player = multiPlayer.players.find(p => p.id === playerId) @@ -1380,13 +1641,18 @@ const label = isPrimary ? `spec ${chLabel}` : `${player?.state?.filename ?? ''} spec ${chLabel}` const id = `${playerId}:spectrogram:ch${ch}` const timeOffset = player?.track?.offsetSec ?? 0 + // maxFreqHz = the tile's true stored ceiling (analysis stores headroom above the view); the + // displayed window is [viewMinHz, maxFreqHz(view)] cropped at render. + const storedCeiling = tile.maxFreqHz ?? spectrogramSettings.maxFreqHz + // Live frequency crop only applies to linear scale (mel rows are non-uniform in Hz). + const viewMinHz = spectrogramSettings.scale === 'mel' ? 0 : spectrogramSettings.viewMinHz if (!mediaSignals.find(s => s.id === id)) { mediaSignals = [ ...mediaSignals, - { id, label, kind: 'spectrogram' as const, imageTimeStart: tile.timeStart, imageTimeEnd: tile.timeEnd, height: 40, maxFreqHz: spectrogramSettings.maxFreqHz, spectrogramDynamicRangeDb: spectrogramSettings.dynamicRangeDb, spectrogramGamma: spectrogramSettings.gamma, timeOffset }, + { id, label, kind: 'spectrogram' as const, imageTimeStart: tile.timeStart, imageTimeEnd: tile.timeEnd, height: 40, maxFreqHz: storedCeiling, viewMinHz, viewMaxHz: spectrogramSettings.maxFreqHz, spectrogramDynamicRangeDb: spectrogramSettings.dynamicRangeDb, spectrogramGamma: spectrogramSettings.gamma, timeOffset }, ].sort(sortSignals) } else { - mediaSignals = mediaSignals.map(s => s.id === id ? { ...s, label, maxFreqHz: spectrogramSettings.maxFreqHz, spectrogramDynamicRangeDb: spectrogramSettings.dynamicRangeDb, spectrogramGamma: spectrogramSettings.gamma } : s) + mediaSignals = mediaSignals.map(s => s.id === id ? { ...s, label, maxFreqHz: storedCeiling, viewMinHz, viewMaxHz: spectrogramSettings.maxFreqHz, spectrogramDynamicRangeDb: spectrogramSettings.dynamicRangeDb, spectrogramGamma: spectrogramSettings.gamma } : s) } _flushSignals() timelineRef?.setSpectrogramOverview(id, tile) @@ -1404,7 +1670,25 @@ ) _flushSignals() }, - onVad(segments) { timelineRef?.setVadSegments(segments); mergedVadSegments = segments }, + onVad(segments) { timelineRef?.setVadSegments(segments); mergedVadSegments = segments; vadComputed = true; vadComputing = false; vadProgress = null }, + onVadProgress(done, total) { + // Final progress clears the running flag even if VAD is disabled (model/WASM unavailable). + if (done >= total && total > 0) { vadProgress = null; vadComputing = false } + else vadProgress = { done, total } + }, + onPitch(playerId, ch, track) { + // Pitch overlays the matching waveform channel. Keep the raw track; the confidence gate is + // applied when building the overlay (see _buildPitchOverlay). + pitchTracks.set(`${playerId}:waveform:ch${ch}`, { times: track.times, f0: track.f0, confidence: track.confidence }) + pitchComputed = true; pitchComputing = false; pitchProgress = null + _syncPitchOverlay() + }, + onPitchProgress(done, total) { + // The final progress (done≥total) marks the pass complete — clear the running flag even if + // no track arrives (pitch disabled: WASM/model unavailable), so it isn't stuck. + if (done >= total && total > 0) { pitchProgress = null; pitchComputing = false } + else pitchProgress = { done, total } + }, onProgress(done, total) { spectrogramProgress = (done >= total && total > 0) ? null : { done, total } }, @@ -1422,12 +1706,34 @@ if (!untrack(() => mediaPreservePitch)) multiPlayer.setPreservePitch(false) function applySpectrogramSettings(newSettings: SpectrogramSettings): void { - const monoChanged = newSettings.monoMix !== spectrogramSettings.monoMix + const old = spectrogramSettings + // The stored ceiling from the last analysis (headroom above the displayed max). If the requested + // view max still fits under it, the frequency window is a pure display crop — no re-decode. + const storedCeiling = Math.max( + old.maxFreqHz, + ...mediaSignals.filter(s => s.kind === 'spectrogram').map(s => s.maxFreqHz ?? 0), + ) + // Fields that require re-analysis (everything except display-only knobs). + const analysisChanged = + newSettings.windowLengthSec !== old.windowLengthSec || + newSettings.hopSec !== old.hopSec || + newSettings.window !== old.window || + newSettings.scale !== old.scale || + newSettings.melBands !== old.melBands || + newSettings.monoMix !== old.monoMix || + newSettings.preEmphasisDbPerOct !== old.preEmphasisDbPerOct || + newSettings.maxFreqHz > storedCeiling + 1 // view max exceeds what we stored → must recompute + + const monoChanged = newSettings.monoMix !== old.monoMix spectrogramSettings = newSettings mediaSignals = mediaSignals .filter(s => !monoChanged || !newSettings.monoMix || !s.id.match(/:(waveform|spectrogram):ch[1-9]/)) - .map(s => s.kind === 'spectrogram' ? { ...s, spectrogramDynamicRangeDb: newSettings.dynamicRangeDb, spectrogramGamma: newSettings.gamma } : s) + .map(s => s.kind === 'spectrogram' + ? { ...s, spectrogramDynamicRangeDb: newSettings.dynamicRangeDb, spectrogramGamma: newSettings.gamma, viewMinHz: newSettings.scale === 'mel' ? 0 : newSettings.viewMinHz, viewMaxHz: newSettings.maxFreqHz } + : s) + _flushSignals() if (!mediaState) return + if (!analysisChanged) return // display-only (dyn range / gamma / frequency window) — instant, no re-decode for (const sig of mediaSignals) { if (sig.kind === 'spectrogram') timelineRef?.clearSpectrogramDetailTiles(sig.id) } @@ -1435,6 +1741,27 @@ multiPlayer.setSpectrogramSettings(newSettings) } + function applyVadSettings(newSettings: VadSettings): void { + vadSettings = newSettings + multiPlayer.setVadSettings(newSettings) // re-segments cached probs instantly (no re-decode) + } + + // Backend + YIN CMNDF threshold re-run pitch (a re-decode). The display filters (confidence gate + // and frequency range) are render-only and re-apply instantly. + function applyPitchDetect(partial: Partial): void { + pitchSettings = { ...pitchSettings, ...partial } + multiPlayer.setPitchSettings(pitchSettings) // remember for the next compute + if (pitchChannels.size > 0 || pitchComputed) { // recompute only if pitch is actually in use + pitchComputed = false; pitchComputing = true; pitchProgress = { done: 0, total: 0 } + multiPlayer.computePitch() + } + } + function applyPitchFilter(partial: Partial): void { + pitchSettings = { ...pitchSettings, ...partial } + multiPlayer.setPitchSettings(pitchSettings) // keep players in sync (no recompute) + _syncPitchOverlay() + } + async function loadMediaFile(file: File, path: string | null = null) { await multiPlayer.loadPrimary(file, path) } @@ -2302,6 +2629,30 @@ function ctxEditUttTier() { } _syncTrackOverlay() } + // Restore persisted pitch. Settings first (so the confidence gate / range match how it was + // computed); then stash the raw f0 tracks keyed by media identity — onWaveform injects each into + // pitchTracks when that channel's waveform signal arrives, so nothing is re-detected. + _pendingLoadedPitch.clear() + pitchChannelSettings.clear() + if (parsed.pitchConfig) { + pitchSettings = _metaToPitchSettings(parsed.pitchConfig.defaults) + multiPlayer.setPitchSettings(pitchSettings) + for (const c of parsed.pitchConfig.channels ?? []) { + pitchChannelSettings.set(_pitchKey(c.mediaKey, c.channelIndex), { mediaKey: c.mediaKey, channelIndex: c.channelIndex, settings: _metaToPitchSettings(c.settings) }) + } + } + for (const entry of unpacked.manifest.pitch ?? []) { + const raw = unpacked.pitch.get(entry.path) + if (!raw) continue + const aligned = raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength) + const all = new Float32Array(aligned) + const n = entry.numFrames + if (all.length < n * 3) continue // truncated / corrupt sidecar + _pendingLoadedPitch.set(_pitchKey(entry.mediaKey, entry.channelIndex), { + times: all.slice(0, n), f0: all.slice(n, n * 2), confidence: all.slice(n * 2, n * 3), + enabled: entry.enabled === true, + }) + } // Media descriptors from the MMEAF, or synthesized from the manifest's // mediaPaths for old .mumo files saved before MEDIA_DESCRIPTOR was embedded. const mediaDescs: EAFMediaDescriptor[] = parsed.media.length > 0 @@ -2377,6 +2728,7 @@ function ctxEditUttTier() { ...(m.mediaHash ? { mediaHash: m.mediaHash } : {}), ...(m.timeOriginMs !== undefined ? { timeOrigin: m.timeOriginMs } : {}), })) } : {}), + pitchConfig: buildPitchConfig(), }, ts) }, }) @@ -4192,6 +4544,7 @@ function ctxEditUttTier() { ...(primaryRel ? { relativeMediaUrl: primaryRel } : {}), ...(primaryTrack?.offsetSec ? { timeOrigin: Math.round(primaryTrack.offsetSec * 1000) } : {}), ...(additionalMedia.length ? { additionalMedia } : {}), + pitchConfig: buildPitchConfig(), }, tokenStore) const imageInputs: MumoImageInput[] = [] @@ -4263,10 +4616,33 @@ function ctxEditUttTier() { } } + // Collect computed pitch tracks (raw f0), keyed by media identity (path/filename) so they reload + // without re-detecting. Persist every computed channel (nothing recomputes after reload). + const pitchInputs: MumoPitchInput[] = [] + for (const [key, track] of pitchTracks) { + const m = /^(.*):waveform:ch(\d+)$/.exec(key) + if (!m) continue + const playerId = m[1]!, ch = Number(m[2]) + const mediaKey = _mediaKey(multiPlayer.players.find(p => p.id === playerId)) + if (!mediaKey) continue // no stable key → skip (would be unrecoverable on reload) + const numFrames = track.f0.length + const buf = new Float32Array(numFrames * 3) + buf.set(track.times, 0) + buf.set(track.f0, numFrames) + buf.set(track.confidence, numFrames * 2) + pitchInputs.push({ + mediaKey, channelIndex: ch, numFrames, + settings: { ...resolvePitchSettings(mediaKey, ch) }, + ...(pitchChannels.has(key) ? { enabled: true } : {}), + data: new Uint8Array(buf.buffer), + }) + } + const packed = packMumo({ mmeaf, images: imageInputs, spectrograms: spectrogramInputs, mediaPaths, ...(trackSetsJSON !== undefined ? { trackSetsJSON } : {}), ...(trackBufferInputs.length ? { trackBuffers: trackBufferInputs } : {}), + ...(pitchInputs.length ? { pitch: pitchInputs } : {}), }) if (isElectron && filecontroller.currentFilePath) { type EApi = { saveFile(path: string, data: Uint8Array): Promise } @@ -4514,9 +4890,11 @@ function ctxEditUttTier() { {participants} {tiers} inUseLabels={participantInUse} + audioChannels={getAudioChannels()} onadd={handleParticipantAdd} onupdate={handleParticipantUpdate} onremove={handleParticipantRemove} + onchannelchange={(id, channel) => { store.setParticipantChannel(id, channel); redrawAllProsody() }} oncopystructure={handleCopyStructure} onclose={() => participantsDlgOpen = false} /> @@ -4919,6 +5297,7 @@ function ctxEditUttTier() { {/if} +
@@ -4958,8 +5337,6 @@ function ctxEditUttTier() {
- -
{#if _showFileOpen}
@@ -5342,6 +5719,9 @@ function ctxEditUttTier() { showEnd={showEndTime} editable={editorMode === 'edit'} getTokenTime={(id) => { const t = store.getTokenTime(id); return (t?.start != null && t?.end != null) ? { start: t.start, end: t.end } : undefined }} + {getIntonation} + {getAudioChannels} + {getParticipantChannel} onEscapeKey={() => setEditorMode('annotate')} tokenClickMode={!!slotFillMode} ontokenhover={handleTokenHover} @@ -5636,6 +6016,34 @@ function ctxEditUttTier() { {#if pct !== null}{pct}%{/if} {/if} + {#if pitchProgress !== null} + {@const ppct = pitchProgress.total > 0 ? Math.round(pitchProgress.done / pitchProgress.total * 100) : null} + + Computing pitch… + + {#if ppct !== null} + + {:else} + + {/if} + + {#if ppct !== null}{ppct}%{/if} + + {/if} + {#if vadProgress !== null} + {@const vpct = vadProgress.total > 0 ? Math.round(vadProgress.done / vadProgress.total * 100) : null} + + Voice activity detection (VAD)… + + {#if vpct !== null} + + {:else} + + {/if} + + {#if vpct !== null}{vpct}%{/if} + + {/if} {#if showFps} {timelineFps} fps {/if} @@ -5658,23 +6066,21 @@ function ctxEditUttTier() { {#if specModalOpen} -