diff --git a/scripts/playwright-issue-48-smoke.mjs b/scripts/playwright-issue-48-smoke.mjs new file mode 100644 index 0000000..429b0fb --- /dev/null +++ b/scripts/playwright-issue-48-smoke.mjs @@ -0,0 +1,139 @@ +import fs from 'node:fs' +import path from 'node:path' +import { spawnSync } from 'node:child_process' + +const DEV_URL = process.env.DEV_URL ?? 'http://localhost:5175/' + +function run(args) { + const result = spawnSync('npx', ['playwright-cli', ...args], { + cwd: process.cwd(), + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024, + }) + if (result.stdout) process.stdout.write(result.stdout) + if (result.stderr) process.stderr.write(result.stderr) + if (result.error) console.error(result.error) + return result.status ?? 1 +} + +const demo = path.resolve('test-fixtures/demo') +const names = ['photo.jpg', 'photo2.jpg'] +const payload = Object.fromEntries( + names.map((name) => [name, fs.readFileSync(path.join(demo, name)).toString('base64')]), +) +payload['slideshow.json'] = Buffer.from(JSON.stringify({ + audioClips: [], + globalSettings: { + fitMode: 'cover', + imageDurationSecs: 3, + kenBurns: false, + transitionType: 'cut', + }, + schemaVersion: 1, + slides: [ + { + durationInFrames: 90, + excluded: false, + filename: 'photo.jpg', + id: 'photo-a', + type: 'image', + }, + { + durationInFrames: 180, + excluded: false, + heading: 'Long title', + id: 'title-b', + kind: 'title', + style: 'dark', + }, + ], +}, null, 2)).toString('base64') + +const code = `async page => { + const payload = ${JSON.stringify(payload)}; + await page.goto(${JSON.stringify(DEV_URL)}); + await page.evaluate((files) => { + function decodeBase64(base64) { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index); + return bytes; + } + class MockFileHandle { + constructor(name, base64) { + this.kind = 'file'; + this.name = name; + this._base64 = base64; + } + async getFile() { + const bytes = decodeBase64(this._base64); + const type = this.name.endsWith('.jpg') ? 'image/jpeg' : 'application/json'; + return new File([bytes], this.name, { type, lastModified: 1781145972852 }); + } + async createWritable() { + const handle = this; + return { + write: async (data) => { + const buffer = data instanceof Blob ? await data.arrayBuffer() : data; + handle._base64 = btoa(String.fromCharCode(...new Uint8Array(buffer))); + }, + close: async () => {}, + }; + } + } + class MockDirHandle { + constructor(files) { + this.name = 'demo'; + this._files = files; + } + async getFileHandle(name, options) { + const handle = this._files.get(name); + if (handle) return handle; + if (options?.create) { + const created = new MockFileHandle(name, ''); + this._files.set(name, created); + return created; + } + throw new DOMException('NotFoundError'); + } + async *values() { + for (const handle of this._files.values()) yield handle; + } + } + const handles = new Map( + Object.entries(files).map(([name, base64]) => [name, new MockFileHandle(name, base64)]), + ); + window.showDirectoryPicker = async () => new MockDirHandle(handles); + }, payload); + + await page.getByRole('button', { name: 'Open Folder' }).click(); + await page.getByText('Long title').first().waitFor(); + + const widths = await page.locator('[data-timeline-block]').evaluateAll((elements) => ( + elements.map((element) => Math.round(element.getBoundingClientRect().width)) + )); + if (widths.length < 2) { + throw new Error('Expected at least two proportional media blocks, got: ' + JSON.stringify(widths)); + } + if (widths[1] <= widths[0]) { + throw new Error('Expected second slide wider than first (3s vs 6s): ' + JSON.stringify(widths)); + } + + const secondSlide = page.locator('[data-timeline-block]').nth(1); + await secondSlide.click(); + await page.waitForTimeout(300); + + const currentHighlight = await secondSlide.evaluate((element) => element.className.includes('ring-emerald-500')); + if (!currentHighlight) { + throw new Error('Expected second slide highlighted as current after click'); + } + + await page.screenshot({ path: 'issue-48-proportional-timeline.png', fullPage: true }); +}` + +let status = run(['open', DEV_URL, '--browser=chrome']) +if (status !== 0) process.exit(status) + +status = run(['run-code', code]) +run(['close']) +process.exit(status) diff --git a/specs/issue-48-proportional-timeline.md b/specs/issue-48-proportional-timeline.md new file mode 100644 index 0000000..14db3ec --- /dev/null +++ b/specs/issue-48-proportional-timeline.md @@ -0,0 +1,39 @@ +# Issue #48 — Proportional timeline UI + audio lane with gain controls + +Closes #48. + +## Scope + +Replace `StoryboardFilmstrip` with a time-proportional `TimelinePanel`: + +- Media lane: slide cards width ∝ first-pass duration from `RenderPlan` +- Audio lane: clip blocks aligned to `audioSegments.startFrame`, waveform peaks, gain sliders (dB) +- Shared horizontal scroll; playhead at `currentFrame` +- Drag-reorder on media lane (slides) and audio lane (clips) +- Gain/reorder removed from sidebar `SoundtrackPanel` (add track + beat grid remain) + +## Pure modules + +### `sequence-planner/timelineLayout.ts` + +- `firstPassEntries(renderPlan)` — entries before loop boundary +- `buildTimelineLayout(renderPlan, pixelsPerFrame, minBlockWidthPx)` → `{ totalWidthPx, mediaBlocks, audioBlocks }` + +### `audio-analysis/waveformPeaks.ts` + +- `computeWaveformPeaks(samples, barCount)` → normalized 0–1 peaks + +## Editor shell + +- `TimelinePanel` — scroll container, playhead, lanes +- `useWaveformPeaks` — decode mono via beat-grid `decodeMono`, cache by filename +- `App` tracks `currentFrame` for playhead; click timeline → seek + +## Testing + +- Unit: `timelineLayout.test.ts`, `waveformPeaks.test.ts` +- Smoke: `scripts/playwright-issue-48-smoke.mjs` — proportional widths, playhead highlight + +## HITL + +Owner review on 10-clip project before merge. diff --git a/specs/multi-track-audio-timeline.md b/specs/multi-track-audio-timeline.md index bcf1597..5a5f6cf 100644 --- a/specs/multi-track-audio-timeline.md +++ b/specs/multi-track-audio-timeline.md @@ -56,3 +56,9 @@ Closes #46. Per-file `beatGridCache` in slideshow.json (migrates legacy single `BeatGrid`). `buildConcatenatedBeatTimes` shifts each clip's beats by clip start. Manual beat grid spans total audio duration. Planner uses position-aware `nudgeSlideEndFrame` when concatenated beat times are provided. `useBeatGrid` analyzes only clips missing from cache; reorder preserves cache. Closes #49. + +## Slice 22 — Proportional timeline UI (in progress) + +`TimelinePanel` replaces `StoryboardFilmstrip`: proportional media widths, audio lane with waveforms and gain sliders, shared scroll + playhead. + +Closes #48. diff --git a/src/audio-analysis/index.ts b/src/audio-analysis/index.ts index 0ca6c29..ace585a 100644 --- a/src/audio-analysis/index.ts +++ b/src/audio-analysis/index.ts @@ -7,4 +7,6 @@ export { TARGET_RMS_DBFS, } from './loudness' export { isLoudnessCacheEntryValid, resolveEffectiveGainDb } from './gain' +export { computeWaveformPeaks, computeWaveformPeakPairs, DEFAULT_WAVEFORM_BAR_COUNT, DEFAULT_WAVEFORM_BUCKET_COUNT } from './waveformPeaks' +export type { WaveformPeakPair } from './waveformPeaks' export type { LoudnessCache, LoudnessCacheEntry } from './types' diff --git a/src/audio-analysis/waveformPeaks.test.ts b/src/audio-analysis/waveformPeaks.test.ts new file mode 100644 index 0000000..c3c69d5 --- /dev/null +++ b/src/audio-analysis/waveformPeaks.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import { + buildSymmetricWaveformPath, + computeWaveformPeakPairs, + computeWaveformPeaks, + resampleWaveformPeaks, +} from './waveformPeaks' + +describe('computeWaveformPeakPairs', () => { + it('returns normalized min/max pairs', () => { + const samples = new Float32Array([0, 0.5, -1, 0.25, 0, 0.75]) + const pairs = computeWaveformPeakPairs(samples, 3) + + expect(pairs).toHaveLength(3) + expect(Math.max(...pairs.map((pair) => Math.max(pair.max, Math.abs(pair.min))))).toBe(1) + expect(Math.min(...pairs.map((pair) => pair.min))).toBeGreaterThanOrEqual(-1) + }) + + it('returns empty array for empty input', () => { + expect(computeWaveformPeakPairs(new Float32Array(), 8)).toEqual([]) + }) +}) + +describe('computeWaveformPeaks', () => { + it('returns normalized peaks between 0 and 1', () => { + const samples = new Float32Array([0, 0.5, -1, 0.25, 0, 0.75]) + const peaks = computeWaveformPeaks(samples, 3) + + expect(peaks).toHaveLength(3) + expect(Math.max(...peaks)).toBeLessThanOrEqual(1) + expect(Math.max(...peaks)).toBeGreaterThan(0) + expect(Math.min(...peaks)).toBeGreaterThanOrEqual(0) + }) +}) + +describe('resampleWaveformPeaks', () => { + it('downsamples to the requested count', () => { + const pairs = computeWaveformPeakPairs(new Float32Array([0, 1, -1, 0.5, -0.5, 0.25]), 6) + expect(resampleWaveformPeaks(pairs, 2)).toHaveLength(2) + }) +}) + +describe('buildSymmetricWaveformPath', () => { + it('returns a closed SVG path', () => { + const pairs = computeWaveformPeakPairs(new Float32Array([0, 1, -1, 0.5, -0.5, 0.25]), 6) + const path = buildSymmetricWaveformPath(pairs, 120, 40) + + expect(path.startsWith('M')).toBe(true) + expect(path.endsWith('Z')).toBe(true) + }) +}) diff --git a/src/audio-analysis/waveformPeaks.ts b/src/audio-analysis/waveformPeaks.ts new file mode 100644 index 0000000..c5d922e --- /dev/null +++ b/src/audio-analysis/waveformPeaks.ts @@ -0,0 +1,116 @@ +export const DEFAULT_WAVEFORM_BUCKET_COUNT = 4096 + +export type WaveformPeakPair = { + max: number + min: number +} + +export function computeWaveformPeakPairs( + samples: Float32Array, + bucketCount = DEFAULT_WAVEFORM_BUCKET_COUNT, +): WaveformPeakPair[] { + if (samples.length === 0 || bucketCount <= 0) return [] + + const pairs: WaveformPeakPair[] = [] + const samplesPerBucket = Math.max(1, Math.floor(samples.length / bucketCount)) + + for (let bucketIndex = 0; bucketIndex < bucketCount; bucketIndex++) { + const start = bucketIndex * samplesPerBucket + const end = bucketIndex === bucketCount - 1 ? samples.length : start + samplesPerBucket + let max = 0 + let min = 0 + + for (let sampleIndex = start; sampleIndex < end; sampleIndex++) { + const sample = samples[sampleIndex] + if (sample > max) max = sample + if (sample < min) min = sample + } + + pairs.push({ max, min }) + } + + const globalPeak = pairs.reduce( + (currentPeak, pair) => Math.max(currentPeak, pair.max, Math.abs(pair.min)), + 0, + ) + if (globalPeak <= 0) return pairs.map(() => ({ max: 0, min: 0 })) + + return pairs.map((pair) => ({ + max: pair.max / globalPeak, + min: pair.min / globalPeak, + })) +} + +/** @deprecated Use computeWaveformPeakPairs for timeline display */ +export const DEFAULT_WAVEFORM_BAR_COUNT = 64 + +/** @deprecated Use computeWaveformPeakPairs for timeline display */ +export function computeWaveformPeaks( + samples: Float32Array, + barCount = DEFAULT_WAVEFORM_BAR_COUNT, +): number[] { + return computeWaveformPeakPairs(samples, barCount).map((pair) => pair.max) +} + +export function resampleWaveformPeaks( + pairs: WaveformPeakPair[], + targetCount: number, +): WaveformPeakPair[] { + if (pairs.length === 0 || targetCount <= 0) return [] + if (pairs.length === targetCount) return pairs + + const resampled: WaveformPeakPair[] = [] + const sourceCount = pairs.length + + for (let targetIndex = 0; targetIndex < targetCount; targetIndex++) { + const sourceStart = Math.floor((targetIndex * sourceCount) / targetCount) + const sourceEnd = Math.max( + sourceStart + 1, + Math.floor(((targetIndex + 1) * sourceCount) / targetCount), + ) + let max = 0 + let min = 0 + + for (let sourceIndex = sourceStart; sourceIndex < sourceEnd; sourceIndex++) { + max = Math.max(max, pairs[sourceIndex].max) + min = Math.min(min, pairs[sourceIndex].min) + } + + resampled.push({ max, min }) + } + + return resampled +} + +export function buildSymmetricWaveformPath( + pairs: WaveformPeakPair[], + width: number, + height: number, +): string { + if (pairs.length === 0 || width <= 0 || height <= 0) return '' + + const sampleCount = Math.max(16, Math.floor(width / 2)) + const resampled = resampleWaveformPeaks(pairs, sampleCount) + const centerY = height / 2 + const halfHeight = (height / 2) * 0.92 + const stepX = width / Math.max(1, resampled.length - 1) + + let path = `M 0 ${centerY}` + + for (let index = 0; index < resampled.length; index++) { + const x = index * stepX + const amplitude = Math.max(resampled[index].max, Math.abs(resampled[index].min)) + path += ` L ${x.toFixed(2)} ${(centerY - amplitude * halfHeight).toFixed(2)}` + } + + path += ` L ${width} ${centerY}` + + for (let index = resampled.length - 1; index >= 0; index--) { + const x = index * stepX + const amplitude = Math.max(resampled[index].max, Math.abs(resampled[index].min)) + path += ` L ${x.toFixed(2)} ${(centerY + amplitude * halfHeight).toFixed(2)}` + } + + path += ' Z' + return path +} diff --git a/src/editor-shell/App.tsx b/src/editor-shell/App.tsx index b9176c4..c1272be 100644 --- a/src/editor-shell/App.tsx +++ b/src/editor-shell/App.tsx @@ -13,6 +13,7 @@ import { import type { GlobalSettings, SlideOverrides, ThemeName } from '../timeline-core' import { applyTheme, dimensionsForAspectRatio } from '../timeline-core' import { plan, slideIdAtFrame, startFrameForSlideId } from '../sequence-planner' +import { resolveEffectiveGainDb } from '../audio-analysis' import { AppHeader } from './AppHeader' import { DropImportLayer } from './DropImportLayer' import { ExportDialog } from './ExportDialog' @@ -20,16 +21,16 @@ import { EditorLayout } from './EditorLayout' import { EditorSidebar } from './EditorSidebar' import { EmptyState } from './EmptyState' import { PlayerPane, FPS } from './PlayerPane' -import { StoryboardFilmstrip } from './StoryboardFilmstrip' +import { TimelinePanel } from './TimelinePanel' import { SlideSettingsDialog } from './SlideSettingsDialog' import { TitleSlideDialog } from './TitleSlideDialog' import { useProject } from './useProject' import { useAudioClipAnalysis } from './useAudioClipAnalysis' import { useBeatGrid } from './useBeatGrid' -import { resolveEffectiveGainDb } from '../audio-analysis' export function App() { const playerRef = useRef(null) + const [currentFrame, setCurrentFrame] = useState(0) const [currentSlideId, setCurrentSlideId] = useState(null) const [selectedSlideId, setSelectedSlideId] = useState(null) const [exporting, setExporting] = useState(false) @@ -154,6 +155,13 @@ export function App() { const canvas = dimensionsForAspectRatio(aspectRatio) const handleFrameChange = useCallback((frame: number) => { + setCurrentFrame(frame) + setCurrentSlideId(slideIdAtFrame(renderPlan, frame)) + }, [renderPlan]) + + const handleSeek = useCallback((frame: number) => { + playerRef.current?.seekTo(frame) + setCurrentFrame(frame) setCurrentSlideId(slideIdAtFrame(renderPlan, frame)) }, [renderPlan]) @@ -231,11 +239,17 @@ export function App() { /> )} filmstrip={slides.length > 0 ? ( - diff --git a/src/editor-shell/SoundtrackPanel.tsx b/src/editor-shell/SoundtrackPanel.tsx index 34e44ba..4c8a62e 100644 --- a/src/editor-shell/SoundtrackPanel.tsx +++ b/src/editor-shell/SoundtrackPanel.tsx @@ -11,7 +11,12 @@ import { } from '@/components/ui/select' import type { AudioTrack } from '../project-store' import type { AudioClip } from '../timeline-core/types' -import { addAudioClip, moveAudioClip, removeAudioClip } from '../timeline-core' +import { + addAudioClip, + moveAudioClip, + removeAudioClip, + updateAudioClipGain, +} from '../timeline-core' import type { BeatGrid } from '../beat-grid/types' import type { LoudnessCache } from '../audio-analysis/types' import { BeatGridPanel } from './BeatGridPanel' @@ -33,14 +38,6 @@ type Props = { onClearManualBeatGrid: () => void } -function updateClipGain(clips: AudioClip[], index: number, gainDb: number | undefined): AudioClip[] { - return clips.map((clip, clipIndex) => { - if (clipIndex !== index) return clip - if (gainDb === undefined) return { filename: clip.filename } - return { filename: clip.filename, gainDb } - }) -} - export function SoundtrackPanel({ analysisStatus, audioClips, @@ -75,7 +72,7 @@ export function SoundtrackPanel({ return (
  • { dragIndexRef.current = null }} onDragOver={(event) => event.preventDefault()} @@ -88,6 +85,7 @@ export function SoundtrackPanel({ dragIndexRef.current = null }} > + {index + 1}. {clip.filename}
    { const raw = event.target.value.trim() if (raw === '') { - onChange(updateClipGain(audioClips, index, undefined)) + onChange(updateAudioClipGain(audioClips, index, undefined)) return } const parsed = Number(raw) if (!Number.isNaN(parsed)) { - onChange(updateClipGain(audioClips, index, parsed)) + onChange(updateAudioClipGain(audioClips, index, parsed)) } }} placeholder={autoGainDb !== undefined ? autoGainDb.toFixed(1) : '0'} @@ -113,7 +111,7 @@ export function SoundtrackPanel({ {clip.gainDb !== undefined ? (
    + ) : audioClips.length === 0 ? ( +

    Drop audio into the project folder, then add it here.

    ) : null} {primaryTrack ? ( diff --git a/src/editor-shell/TimelineAudioClip.tsx b/src/editor-shell/TimelineAudioClip.tsx new file mode 100644 index 0000000..308ca7d --- /dev/null +++ b/src/editor-shell/TimelineAudioClip.tsx @@ -0,0 +1,62 @@ +import { cn } from '@/lib/utils' +import type { WaveformPeakPair } from '../audio-analysis' +import type { TimelineAudioBlock } from '../sequence-planner' +import { TimelineWaveform } from './TimelineWaveform' + +const WAVEFORM_HEIGHT_PX = 56 + +type Props = { + autoGainDb: number | undefined + clipIndex: number + manualGainDb: number | undefined + peaks: WaveformPeakPair[] | undefined + segment: TimelineAudioBlock +} + +export function TimelineAudioClip({ + autoGainDb, + clipIndex, + manualGainDb, + peaks, + segment, +}: Props) { + const displayGainDb = manualGainDb ?? autoGainDb ?? 0 + const clipWidthPx = Math.max(1, Math.floor(segment.widthPx - 2)) + + return ( +
    0 && 'border-l-2 border-l-amber-300/90', + clipIndex === 0 && 'border-l border-l-emerald-600/50', + 'border-r border-r-emerald-600/50', + )} + data-timeline-block="" + style={{ left: segment.leftPx, width: segment.widthPx }} + > +
    + + + {clipIndex + 1} + +
    +
    + {segment.filename} + + {displayGainDb.toFixed(1)} dB + +
    +
    + ) +} diff --git a/src/editor-shell/TimelineMediaBlock.tsx b/src/editor-shell/TimelineMediaBlock.tsx new file mode 100644 index 0000000..b3db102 --- /dev/null +++ b/src/editor-shell/TimelineMediaBlock.tsx @@ -0,0 +1,119 @@ +import type { MutableRefObject } from 'react' +import { cn } from '@/lib/utils' +import type { Slide } from '../timeline-core/types' +import { isTitleSlide } from '../timeline-core/types' + +type Props = { + currentSlideId: string | null + dragIndexRef: MutableRefObject + leftPx: number + onReorder: (fromIndex: number, toIndex: number) => void + onSlideClick: (id: string) => void + onToggleExclude: (id: string) => void + selectedSlideId: string | null + slide: Slide + slideIndex: number + widthPx: number +} + +function hasOverrides(slide: Slide): boolean { + return !!slide.overrides && Object.keys(slide.overrides).length > 0 +} + +function slideLabel(slide: Slide): string { + return isTitleSlide(slide) ? slide.heading || 'Title' : slide.filename +} + +export function TimelineMediaBlock({ + currentSlideId, + dragIndexRef, + leftPx, + onReorder, + onSlideClick, + onToggleExclude, + selectedSlideId, + slide, + slideIndex, + widthPx, +}: Props) { + return ( +
  • onSlideClick(slide.id)} + onDragEnd={() => { dragIndexRef.current = null }} + onDragOver={(event) => event.preventDefault()} + onDragStart={() => { dragIndexRef.current = slideIndex }} + onDrop={(event) => { + event.preventDefault() + event.stopPropagation() + if (dragIndexRef.current !== null && dragIndexRef.current !== slideIndex) { + onReorder(dragIndexRef.current, slideIndex) + } + dragIndexRef.current = null + }} + style={{ left: leftPx, width: widthPx }} + > +
    + {isTitleSlide(slide) ? ( +
    + {slide.heading.slice(0, 14) || 'T'} +
    + ) : slide.type === 'video' ? ( +
    + {slideLabel(slide)} +
    • + ) +} diff --git a/src/editor-shell/TimelinePanel.tsx b/src/editor-shell/TimelinePanel.tsx new file mode 100644 index 0000000..812967a --- /dev/null +++ b/src/editor-shell/TimelinePanel.tsx @@ -0,0 +1,174 @@ +import { useCallback, useEffect, useMemo, useRef } from 'react' +import type { AudioClip } from '../timeline-core/types' +import type { Slide } from '../timeline-core/types' +import type { LoudnessCache } from '../audio-analysis/types' +import { + buildTimelineLayout, + type RenderPlan, +} from '../sequence-planner' +import type { AudioTrack } from '../project-store' +import { TimelineAudioClip } from './TimelineAudioClip' +import { TimelineMediaBlock } from './TimelineMediaBlock' +import { TimelineZoomControls } from './TimelineZoomControls' +import { useTimelineZoom } from './useTimelineZoom' +import { useWaveformPeaks } from './useWaveformPeaks' + +type Props = { + audioClips: AudioClip[] + audioTracks: AudioTrack[] + currentFrame: number + currentSlideId: string | null + loudnessCache: LoudnessCache | undefined + onReorder: (fromIndex: number, toIndex: number) => void + onSeek: (frame: number) => void + onSlideClick: (id: string) => void + onToggleExclude: (id: string) => void + renderPlan: RenderPlan + selectedSlideId: string | null + slides: Slide[] +} + +export function TimelinePanel({ + audioClips, + audioTracks, + currentFrame, + currentSlideId, + loudnessCache, + onReorder, + onSeek, + onSlideClick, + onToggleExclude, + renderPlan, + selectedSlideId, + slides, +}: Props) { + const scrollRef = useRef(null) + const mediaDragIndexRef = useRef(null) + const { waveformCache } = useWaveformPeaks({ audioClips, audioTracks }) + const { + pixelsPerFrame, + resetZoom, + setPixelsPerFrame, + zoomIn, + zoomOut, + zoomPercent, + } = useTimelineZoom({ scrollRef }) + + const audioFilenames = useMemo( + () => audioClips.map((clip) => clip.filename), + [audioClips], + ) + + const layout = useMemo( + () => buildTimelineLayout(slides, renderPlan, audioFilenames, pixelsPerFrame), + [audioFilenames, pixelsPerFrame, renderPlan, slides], + ) + + const playheadLeftPx = currentFrame * pixelsPerFrame + const included = slides.filter((slide) => !slide.excluded).length + + const handleTimelineClick = useCallback((event: React.MouseEvent) => { + if ((event.target as HTMLElement).closest('[data-timeline-block]')) return + + const bounds = event.currentTarget.getBoundingClientRect() + const scrollLeft = scrollRef.current?.scrollLeft ?? 0 + const clickX = event.clientX - bounds.left + scrollLeft + const frame = Math.round(clickX / pixelsPerFrame) + const clampedFrame = Math.max(0, Math.min(frame, Math.max(renderPlan.totalFrames - 1, 0))) + onSeek(clampedFrame) + }, [onSeek, pixelsPerFrame, renderPlan.totalFrames]) + + useEffect(() => { + const scrollElement = scrollRef.current + if (!scrollElement) return + + const playheadX = playheadLeftPx + const viewStart = scrollElement.scrollLeft + const viewEnd = viewStart + scrollElement.clientWidth + const margin = 48 + + if (playheadX < viewStart + margin || playheadX > viewEnd - margin) { + scrollElement.scrollLeft = Math.max(0, playheadX - scrollElement.clientWidth / 2) + } + }, [playheadLeftPx]) + + return ( +
    +
    +

    + {included === slides.length + ? `${slides.length} slide${slides.length !== 1 ? 's' : ''}` + : `${included} / ${slides.length} included`} + · ⌘/ctrl + scroll to zoom +

    + +
    +
    +
    +
    + +
    +

    Media

    +
      + {slides.map((slide, slideIndex) => { + const block = layout.mediaBlocks[slideIndex] + if (!block) return null + + return ( + + ) + })} +
    +
    + + {layout.audioBlocks.length > 0 ? ( +
    +

    + Audio · reorder and gain in Soundtrack sidebar +

    +
    + {layout.audioBlocks.map((segment, clipIndex) => ( + + ))} +
    +
    + ) : null} +
    +
    +
    + ) +} diff --git a/src/editor-shell/TimelineWaveform.tsx b/src/editor-shell/TimelineWaveform.tsx new file mode 100644 index 0000000..cc491f4 --- /dev/null +++ b/src/editor-shell/TimelineWaveform.tsx @@ -0,0 +1,39 @@ +import { useMemo } from 'react' +import type { WaveformPeakPair } from '../audio-analysis' +import { buildSymmetricWaveformPath } from '../audio-analysis/waveformPeaks' + +type Props = { + height: number + peaks: WaveformPeakPair[] | undefined + width: number +} + +export function TimelineWaveform({ height, peaks, width }: Props) { + const path = useMemo(() => { + if (!peaks || peaks.length === 0 || width <= 0 || height <= 0) return '' + return buildSymmetricWaveformPath(peaks, width, height) + }, [height, peaks, width]) + + if (!path) { + return
    + } + + return ( + + + + ) +} diff --git a/src/editor-shell/TimelineZoomControls.tsx b/src/editor-shell/TimelineZoomControls.tsx new file mode 100644 index 0000000..4dfa670 --- /dev/null +++ b/src/editor-shell/TimelineZoomControls.tsx @@ -0,0 +1,71 @@ +import { Minus, Plus } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Slider } from '@/components/ui/slider' +import { + MAX_PIXELS_PER_FRAME, + MIN_PIXELS_PER_FRAME, + TIMELINE_ZOOM_STEP, +} from '../sequence-planner' + +type Props = { + onResetZoom: () => void + onZoomChange: (pixelsPerFrame: number) => void + onZoomIn: () => void + onZoomOut: () => void + pixelsPerFrame: number + zoomPercent: number +} + +export function TimelineZoomControls({ + onResetZoom, + onZoomChange, + onZoomIn, + onZoomOut, + pixelsPerFrame, + zoomPercent, +}: Props) { + return ( +
    + + { + const nextValue = values[0] + if (nextValue !== undefined) onZoomChange(nextValue) + }} + step={TIMELINE_ZOOM_STEP} + value={[pixelsPerFrame]} + /> + + +
    + ) +} diff --git a/src/editor-shell/useTimelineZoom.ts b/src/editor-shell/useTimelineZoom.ts new file mode 100644 index 0000000..7066aac --- /dev/null +++ b/src/editor-shell/useTimelineZoom.ts @@ -0,0 +1,87 @@ +import { useCallback, useEffect, useState } from 'react' +import { + DEFAULT_PIXELS_PER_FRAME, + MAX_PIXELS_PER_FRAME, + MIN_PIXELS_PER_FRAME, + TIMELINE_ZOOM_STEP, +} from '../sequence-planner' + +function clampPixelsPerFrame(value: number): number { + return Math.min(MAX_PIXELS_PER_FRAME, Math.max(MIN_PIXELS_PER_FRAME, value)) +} + +type Options = { + scrollRef: React.RefObject +} + +export function useTimelineZoom({ scrollRef }: Options) { + const [pixelsPerFrame, setPixelsPerFrameState] = useState(DEFAULT_PIXELS_PER_FRAME) + + const setPixelsPerFrame = useCallback((nextValue: number | ((previous: number) => number)) => { + setPixelsPerFrameState((previousValue) => { + const resolvedValue = clampPixelsPerFrame( + typeof nextValue === 'function' ? nextValue(previousValue) : nextValue, + ) + + if (resolvedValue === previousValue) return previousValue + + const scrollElement = scrollRef.current + const centerScroll = scrollElement + ? scrollElement.scrollLeft + scrollElement.clientWidth / 2 + : null + const zoomRatio = resolvedValue / previousValue + + if (centerScroll !== null) { + requestAnimationFrame(() => { + const element = scrollRef.current + if (!element) return + element.scrollLeft = Math.max( + 0, + centerScroll * zoomRatio - element.clientWidth / 2, + ) + }) + } + + return resolvedValue + }) + }, [scrollRef]) + + const zoomIn = useCallback(() => { + setPixelsPerFrame((previous) => previous + TIMELINE_ZOOM_STEP) + }, [setPixelsPerFrame]) + + const zoomOut = useCallback(() => { + setPixelsPerFrame((previous) => previous - TIMELINE_ZOOM_STEP) + }, [setPixelsPerFrame]) + + const resetZoom = useCallback(() => { + setPixelsPerFrame(DEFAULT_PIXELS_PER_FRAME) + }, [setPixelsPerFrame]) + + useEffect(() => { + const scrollElement = scrollRef.current + if (!scrollElement) return + + function handleWheel(event: WheelEvent) { + if (!event.ctrlKey && !event.metaKey) return + event.preventDefault() + + const factor = event.deltaY > 0 ? 0.9 : 1.1 + setPixelsPerFrame((previous) => previous * factor) + } + + scrollElement.addEventListener('wheel', handleWheel, { passive: false }) + return () => scrollElement.removeEventListener('wheel', handleWheel) + }, [scrollRef, setPixelsPerFrame]) + + const zoomPercent = Math.round((pixelsPerFrame / DEFAULT_PIXELS_PER_FRAME) * 100) + + return { + pixelsPerFrame, + resetZoom, + setPixelsPerFrame, + zoomIn, + zoomOut, + zoomPercent, + } +} diff --git a/src/editor-shell/useWaveformPeaks.ts b/src/editor-shell/useWaveformPeaks.ts new file mode 100644 index 0000000..163c3fe --- /dev/null +++ b/src/editor-shell/useWaveformPeaks.ts @@ -0,0 +1,73 @@ +import { useEffect, useMemo, useState } from 'react' +import type { AudioClip } from '../timeline-core/types' +import type { AudioTrack } from '../project-store' +import { decodeMono } from '../beat-grid' +import { computeWaveformPeakPairs, type WaveformPeakPair } from '../audio-analysis' + +type WaveformCache = Record + +type Options = { + audioClips: AudioClip[] + audioTracks: AudioTrack[] +} + +function tracksForClips(audioClips: AudioClip[], audioTracks: AudioTrack[]): AudioTrack[] { + return audioClips + .map((clip) => audioTracks.find((track) => track.filename === clip.filename)) + .filter((track): track is AudioTrack => track !== undefined) +} + +export function useWaveformPeaks({ audioClips, audioTracks }: Options) { + const [waveformCache, setWaveformCache] = useState({}) + + const playlistTracks = useMemo( + () => tracksForClips(audioClips, audioTracks), + [audioClips, audioTracks], + ) + + const pendingFilenames = useMemo( + () => playlistTracks + .map((track) => track.filename) + .filter((filename) => waveformCache[filename] === undefined), + [playlistTracks, waveformCache], + ) + + useEffect(() => { + if (pendingFilenames.length === 0) return + + let cancelled = false + + async function decodePeaks() { + const updates: WaveformCache = {} + + for (const filename of pendingFilenames) { + if (cancelled) return + + const track = playlistTracks.find((entry) => entry.filename === filename) + if (!track) continue + + try { + const response = await fetch(track.blobUrl) + const buffer = await response.arrayBuffer() + const { samples } = await decodeMono(buffer) + if (cancelled) return + updates[filename] = computeWaveformPeakPairs(samples) + } catch { + if (cancelled) return + updates[filename] = [] + } + } + + if (cancelled || Object.keys(updates).length === 0) return + setWaveformCache((previous) => ({ ...previous, ...updates })) + } + + const deferId = window.setTimeout(() => { void decodePeaks() }, 0) + return () => { + cancelled = true + window.clearTimeout(deferId) + } + }, [pendingFilenames, playlistTracks]) + + return { waveformCache } +} diff --git a/src/sequence-planner/index.ts b/src/sequence-planner/index.ts index d5da45d..5d5efad 100644 --- a/src/sequence-planner/index.ts +++ b/src/sequence-planner/index.ts @@ -1,5 +1,16 @@ export { plan, TRANSITION_FRAMES } from './planner' export { slideIdAtFrame, startFrameForSlideId } from './playback' +export { + buildTimelineLayout, + DEFAULT_MIN_BLOCK_WIDTH_PX, + DEFAULT_PIXELS_PER_FRAME, + firstPassEntries, + MAX_PIXELS_PER_FRAME, + MIN_PIXELS_PER_FRAME, + TIMELINE_BLOCK_GAP_PX, + TIMELINE_ZOOM_STEP, +} from './timelineLayout' +export type { TimelineAudioBlock, TimelineLayout, TimelineMediaBlock } from './timelineLayout' export type { AudioClipInput } from './planner' export type { AudioSegment, diff --git a/src/sequence-planner/timelineLayout.test.ts b/src/sequence-planner/timelineLayout.test.ts new file mode 100644 index 0000000..f7ac221 --- /dev/null +++ b/src/sequence-planner/timelineLayout.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from 'vitest' +import type { RenderPlan } from './types' +import { + buildTimelineLayout, + DEFAULT_MIN_BLOCK_WIDTH_PX, + DEFAULT_PIXELS_PER_FRAME, + firstPassEntries, +} from './timelineLayout' + +function imageEntry(id: string, startFrame: number, durationInFrames: number): RenderPlan['entries'][number] { + return { + durationInFrames, + fitMode: 'cover', + kenBurns: null, + slide: { + blobUrl: `blob:${id}`, + durationInFrames, + excluded: false, + filename: `${id}.jpg`, + id, + type: 'image', + }, + startFrame, + videoVolume: 0, + } +} + +describe('firstPassEntries', () => { + it('returns all entries when there is no loop', () => { + const renderPlan: RenderPlan = { + entries: [ + imageEntry('a', 0, 90), + imageEntry('b', 75, 120), + ], + totalFrames: 195, + } + + expect(firstPassEntries(renderPlan)).toHaveLength(2) + }) + + it('stops at the second occurrence of the first slide id', () => { + const renderPlan: RenderPlan = { + entries: [ + imageEntry('a', 0, 90), + imageEntry('b', 75, 90), + imageEntry('a', 150, 90), + ], + totalFrames: 240, + } + + expect(firstPassEntries(renderPlan)).toHaveLength(2) + }) +}) + +describe('buildTimelineLayout', () => { + it('assigns proportional widths to media blocks', () => { + const renderPlan: RenderPlan = { + entries: [ + imageEntry('a', 0, 60), + imageEntry('b', 45, 120), + ], + totalFrames: 165, + } + + const layout = buildTimelineLayout( + [ + renderPlan.entries[0].slide, + renderPlan.entries[1].slide, + ], + renderPlan, + [], + ) + + expect(layout.mediaBlocks[0].widthPx).toBe(60 * DEFAULT_PIXELS_PER_FRAME) + expect(layout.mediaBlocks[1].widthPx).toBe(120 * DEFAULT_PIXELS_PER_FRAME) + expect(layout.totalWidthPx).toBeGreaterThanOrEqual(165 * DEFAULT_PIXELS_PER_FRAME) + }) + + it('enforces a minimum block width for very short slides', () => { + const renderPlan: RenderPlan = { + entries: [imageEntry('a', 0, 5)], + totalFrames: 5, + } + + const layout = buildTimelineLayout([renderPlan.entries[0].slide], renderPlan, []) + + expect(layout.mediaBlocks[0].widthPx).toBe(DEFAULT_MIN_BLOCK_WIDTH_PX) + }) + + it('places audio blocks at segment start frames', () => { + const renderPlan: RenderPlan = { + audioSegments: [ + { + blobUrl: 'blob:one', + durationInFrames: 90, + gainDb: 0, + startFrame: 0, + }, + { + blobUrl: 'blob:two', + durationInFrames: 60, + gainDb: -3, + startFrame: 90, + }, + ], + entries: [imageEntry('a', 0, 90)], + totalFrames: 150, + } + + const layout = buildTimelineLayout([renderPlan.entries[0].slide], renderPlan, ['one.mp3', 'two.mp3']) + + expect(layout.audioBlocks).toEqual([ + expect.objectContaining({ + filename: 'one.mp3', + leftPx: 0, + startFrame: 0, + widthPx: 90 * DEFAULT_PIXELS_PER_FRAME, + }), + expect.objectContaining({ + filename: 'two.mp3', + gainDb: -3, + leftPx: 90 * DEFAULT_PIXELS_PER_FRAME, + startFrame: 90, + widthPx: 60 * DEFAULT_PIXELS_PER_FRAME, + }), + ]) + }) +}) diff --git a/src/sequence-planner/timelineLayout.ts b/src/sequence-planner/timelineLayout.ts new file mode 100644 index 0000000..1d7439e --- /dev/null +++ b/src/sequence-planner/timelineLayout.ts @@ -0,0 +1,125 @@ +import { isTitleSlide } from '../timeline-core/types' +import type { Slide } from '../timeline-core/types' +import type { RenderPlan, RenderPlanEntry } from './types' + +export const DEFAULT_PIXELS_PER_FRAME = 2 +export const MAX_PIXELS_PER_FRAME = 10 +export const MIN_PIXELS_PER_FRAME = 0.5 +export const TIMELINE_ZOOM_STEP = 0.25 +export const DEFAULT_MIN_BLOCK_WIDTH_PX = 40 +export const TIMELINE_BLOCK_GAP_PX = 4 + +export type TimelineMediaBlock = { + durationInFrames: number + leftPx: number + slideId: string + widthPx: number +} + +export type TimelineAudioBlock = { + blobUrl: string + durationInFrames: number + filename: string + gainDb: number + leftPx: number + startFrame: number + widthPx: number +} + +export type TimelineLayout = { + audioBlocks: TimelineAudioBlock[] + mediaBlocks: TimelineMediaBlock[] + totalWidthPx: number +} + +export function firstPassEntries(renderPlan: RenderPlan): RenderPlanEntry[] { + if (renderPlan.entries.length === 0) return [] + + const firstSlideId = renderPlan.entries[0].slide.id + const pass: RenderPlanEntry[] = [] + + for (const entry of renderPlan.entries) { + if (pass.length > 0 && entry.slide.id === firstSlideId) break + pass.push(entry) + } + + return pass +} + +function blockWidthPx( + durationInFrames: number, + pixelsPerFrame: number, + minBlockWidthPx: number, +): number { + return Math.max(minBlockWidthPx, durationInFrames * pixelsPerFrame) +} + +function durationInFramesForSlide(slide: Slide, renderPlan: RenderPlan): number { + const entry = firstPassEntries(renderPlan).find((planEntry) => planEntry.slide.id === slide.id) + if (entry) return entry.durationInFrames + if (isTitleSlide(slide)) return slide.durationInFrames + return slide.durationInFrames +} + +function buildMediaBlocks( + slides: Slide[], + renderPlan: RenderPlan, + pixelsPerFrame: number, + minBlockWidthPx: number, +): TimelineMediaBlock[] { + let leftPx = 0 + const blocks: TimelineMediaBlock[] = [] + + for (const slide of slides) { + const durationInFrames = durationInFramesForSlide(slide, renderPlan) + const widthPx = blockWidthPx(durationInFrames, pixelsPerFrame, minBlockWidthPx) + blocks.push({ + durationInFrames, + leftPx, + slideId: slide.id, + widthPx, + }) + leftPx += widthPx + TIMELINE_BLOCK_GAP_PX + } + + return blocks +} + +export function buildTimelineLayout( + slides: Slide[], + renderPlan: RenderPlan, + audioFilenames: string[], + pixelsPerFrame = DEFAULT_PIXELS_PER_FRAME, + minBlockWidthPx = DEFAULT_MIN_BLOCK_WIDTH_PX, +): TimelineLayout { + const mediaBlocks = buildMediaBlocks(slides, renderPlan, pixelsPerFrame, minBlockWidthPx) + const mediaContentWidthPx = mediaBlocks.length > 0 + ? mediaBlocks[mediaBlocks.length - 1].leftPx + + mediaBlocks[mediaBlocks.length - 1].widthPx + : 0 + const audioEndPx = (renderPlan.audioSegments ?? []).reduce( + (maxEnd, segment) => Math.max( + maxEnd, + (segment.startFrame + segment.durationInFrames) * pixelsPerFrame, + ), + 0, + ) + const totalWidthPx = Math.max( + minBlockWidthPx, + renderPlan.totalFrames * pixelsPerFrame, + mediaContentWidthPx, + audioEndPx, + ) + + const audioBlocks: TimelineAudioBlock[] = (renderPlan.audioSegments ?? []).map((segment, index) => ({ + blobUrl: segment.blobUrl, + durationInFrames: segment.durationInFrames, + filename: audioFilenames[index] ?? segment.blobUrl, + gainDb: segment.gainDb, + leftPx: segment.startFrame * pixelsPerFrame, + startFrame: segment.startFrame, + widthPx: blockWidthPx(segment.durationInFrames, pixelsPerFrame, minBlockWidthPx), + })) + + return { audioBlocks, mediaBlocks, totalWidthPx } +} diff --git a/src/timeline-core/audioClips.test.ts b/src/timeline-core/audioClips.test.ts index df0b90d..ee96970 100644 --- a/src/timeline-core/audioClips.test.ts +++ b/src/timeline-core/audioClips.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { addAudioClip, moveAudioClip, removeAudioClip } from './audioClips' +import { addAudioClip, moveAudioClip, removeAudioClip, updateAudioClipGain } from './audioClips' import type { AudioClip } from './types' const CLIPS: AudioClip[] = [ @@ -31,4 +31,17 @@ describe('audio clip ordering', () => { { filename: 'c.mp3' }, ]) }) + + it('updates manual gain and clears it when undefined', () => { + expect(updateAudioClipGain(CLIPS, 1, -3)).toEqual([ + { filename: 'a.mp3' }, + { filename: 'b.mp3', gainDb: -3 }, + { filename: 'c.mp3' }, + ]) + expect(updateAudioClipGain( + [{ filename: 'b.mp3', gainDb: -3 }], + 0, + undefined, + )).toEqual([{ filename: 'b.mp3' }]) + }) }) diff --git a/src/timeline-core/audioClips.ts b/src/timeline-core/audioClips.ts index 87e8950..d685b4b 100644 --- a/src/timeline-core/audioClips.ts +++ b/src/timeline-core/audioClips.ts @@ -16,3 +16,15 @@ export function moveAudioClip(clips: AudioClip[], fromIndex: number, toIndex: nu export function removeAudioClip(clips: AudioClip[], index: number): AudioClip[] { return clips.filter((_, clipIndex) => clipIndex !== index) } + +export function updateAudioClipGain( + clips: AudioClip[], + index: number, + gainDb: number | undefined, +): AudioClip[] { + return clips.map((clip, clipIndex) => { + if (clipIndex !== index) return clip + if (gainDb === undefined) return { filename: clip.filename } + return { filename: clip.filename, gainDb } + }) +} diff --git a/src/timeline-core/index.ts b/src/timeline-core/index.ts index 337b5c2..641b8a9 100644 --- a/src/timeline-core/index.ts +++ b/src/timeline-core/index.ts @@ -1,7 +1,7 @@ export { ASPECT_RATIOS, DEFAULT_ASPECT_RATIO, dimensionsForAspectRatio, isAspectRatio } from './aspect' export type { AspectRatio, CanvasDimensions } from './aspect' export { getMediaType, isSupportedAudio, isSupportedMedia, sortByFilename } from './media' -export { addAudioClip, moveAudioClip, removeAudioClip } from './audioClips' +export { addAudioClip, moveAudioClip, removeAudioClip, updateAudioClipGain } from './audioClips' export { moveSlide, toggleExcluded, filterIncluded, createTitleSlide } from './timeline' export { resolve, applyImageDuration, DEFAULT_GLOBAL_SETTINGS, THEMES, applyTheme } from './settings' export type {