From 10730c08f14a1245feb485dd3ad2d42cc3edbb06 Mon Sep 17 00:00:00 2001 From: cbaucom Date: Mon, 22 Jun 2026 16:42:36 -0400 Subject: [PATCH 1/2] Add smart-fit as default photo fit mode. Resolve fit per slide from media vs canvas aspect ratio so mismatched orientations letterbox instead of crop, while keeping cover for compatible pairs and manual overrides. Co-authored-by: Cursor --- specs/smart-fit.md | 31 ++++++++++ src/editor-shell/App.tsx | 3 +- src/editor-shell/GlobalSettingsPanel.tsx | 1 + src/editor-shell/SlideSettingsDialog.tsx | 1 + src/project-store/media-loader.ts | 29 +++++++++ src/sequence-planner/planner.test.ts | 35 ++++++++++- src/sequence-planner/planner.ts | 30 +++++++++- src/sequence-planner/types.ts | 7 ++- src/timeline-core/index.ts | 2 + src/timeline-core/settings.ts | 10 ++-- src/timeline-core/smartFit.test.ts | 76 ++++++++++++++++++++++++ src/timeline-core/smartFit.ts | 46 ++++++++++++++ src/timeline-core/types.ts | 4 ++ 13 files changed, 262 insertions(+), 13 deletions(-) create mode 100644 specs/smart-fit.md create mode 100644 src/timeline-core/smartFit.test.ts create mode 100644 src/timeline-core/smartFit.ts diff --git a/specs/smart-fit.md b/specs/smart-fit.md new file mode 100644 index 0000000..c430270 --- /dev/null +++ b/specs/smart-fit.md @@ -0,0 +1,31 @@ +# Smart Fit — Technical Spec + +## Problem + +Default `cover` fit crops media when its aspect ratio differs from the project canvas (e.g. 16:9 photo in a 9:16 project). Users expect the full image to remain visible unless they explicitly choose crop. + +## Behavior + +- **`smart-fit`** becomes the default global `fitMode` (themes included). +- At plan time, `smart-fit` resolves to a concrete fit per slide: + - **Orientation mismatch** (landscape ↔ portrait) or **square ↔ non-square** → `blur-fill` (full media visible, blurred background). + - **Compatible orientation** (both landscape, both portrait, or both square) → `cover` (Ken Burns-friendly crop-fill). + - **Missing media dimensions** → `contain` (safe fallback, no crop). +- **Videos** remain always `contain` (unchanged). +- **Per-slide overrides** (`cover`, `contain`, `blur-fill`, `smart-fit`) still win via the settings cascade. +- **Persisted `fitMode: 'cover'`** in existing projects is unchanged until the user switches. + +## Architecture + +| Layer | Responsibility | +| --- | --- | +| `timeline-core/smartFit.ts` | Pure `resolveSmartFit(mediaW, mediaH, canvasW, canvasH)` | +| `timeline-core/settings.ts` | Add `'smart-fit'` to `FitMode`, default + themes | +| `project-store/media-loader.ts` | Extract `width`/`height` at import (image bitmap, video track) | +| `sequence-planner/planner.ts` | Accept `aspectRatio`, resolve `smart-fit` using slide dims + canvas | +| `composition/` | No changes — receives resolved fit modes only | + +## Tests + +- `smartFit.test.ts`: orientation pairs, square cases, missing/zero dimensions. +- `planner.test.ts`: smart-fit resolution with metadata + aspect ratio; overrides still win. diff --git a/src/editor-shell/App.tsx b/src/editor-shell/App.tsx index c1272be..193d254 100644 --- a/src/editor-shell/App.tsx +++ b/src/editor-shell/App.tsx @@ -148,8 +148,9 @@ export function App() { planAudioClips.length > 0 ? planAudioClips : undefined, beatGrid.effectiveBeatGrid, planBeatTimes, + aspectRatio, ), - [beatGrid.effectiveBeatGrid, deferredGlobalSettings, deferredSlides, planAudioClips, planBeatTimes], + [aspectRatio, beatGrid.effectiveBeatGrid, deferredGlobalSettings, deferredSlides, planAudioClips, planBeatTimes], ) const totalFrames = renderPlan.totalFrames > 0 ? renderPlan.totalFrames : FPS const canvas = dimensionsForAspectRatio(aspectRatio) diff --git a/src/editor-shell/GlobalSettingsPanel.tsx b/src/editor-shell/GlobalSettingsPanel.tsx index 7de8407..267fa28 100644 --- a/src/editor-shell/GlobalSettingsPanel.tsx +++ b/src/editor-shell/GlobalSettingsPanel.tsx @@ -170,6 +170,7 @@ export function GlobalSettingsPanel({ aspectRatio, onAspectRatioChange, settings + Smart fit Cover (crop) Letterbox Blur fill diff --git a/src/editor-shell/SlideSettingsDialog.tsx b/src/editor-shell/SlideSettingsDialog.tsx index 5f813cc..cec501a 100644 --- a/src/editor-shell/SlideSettingsDialog.tsx +++ b/src/editor-shell/SlideSettingsDialog.tsx @@ -137,6 +137,7 @@ export function SlideSettingsDialog({ globalSettings, onClose, onOverride, slide + Smart fit Cover (crop) Letterbox Blur fill diff --git a/src/project-store/media-loader.ts b/src/project-store/media-loader.ts index c77872a..9fc23a2 100644 --- a/src/project-store/media-loader.ts +++ b/src/project-store/media-loader.ts @@ -25,6 +25,31 @@ async function getVideoDurationFrames(file: File): Promise { } } +async function getImageDimensions(file: File): Promise<{ height: number; width: number } | undefined> { + try { + const bitmap = await createImageBitmap(file) + const dimensions = { height: bitmap.height, width: bitmap.width } + bitmap.close() + return dimensions + } catch { + return undefined + } +} + +async function getVideoDimensions(file: File): Promise<{ height: number; width: number } | undefined> { + try { + const input = new Input({ + formats: ALL_FORMATS, + source: new BlobSource(file), + }) + const videoTrack = await input.getPrimaryVideoTrack() + if (!videoTrack) return undefined + return { height: videoTrack.displayHeight, width: videoTrack.displayWidth } + } catch { + return undefined + } +} + export async function createMediaSlideFromFile(file: File): Promise { const filename = file.name const type = getMediaType(filename) @@ -33,12 +58,16 @@ export async function createMediaSlideFromFile(file: File): Promise type === 'video' ? await getVideoDurationFrames(file) : IMAGE_DURATION_FRAMES + const dimensions = type === 'video' + ? await getVideoDimensions(file) + : await getImageDimensions(file) return { blobUrl, durationInFrames, excluded: false, filename, + ...(dimensions ? { height: dimensions.height, width: dimensions.width } : {}), id: `${filename}-${file.lastModified}`, type, } diff --git a/src/sequence-planner/planner.test.ts b/src/sequence-planner/planner.test.ts index 0193dcb..20d38a0 100644 --- a/src/sequence-planner/planner.test.ts +++ b/src/sequence-planner/planner.test.ts @@ -261,6 +261,13 @@ describe('plan — Ken Burns vectors', () => { describe('plan — fit mode resolution', () => { const IMG = makeSlide('i', 'image', 90) const VID = makeSlide('v', 'video', 120) + const LANDSCAPE_IMG: MediaSlide = { + ...IMG, + filename: 'landscape.jpg', + height: 1080, + width: 1920, + } + const PORTRAIT_CANVAS = '9:16' as const it('videos always get fitMode=contain regardless of global setting', () => { const settings: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, fitMode: 'cover' } @@ -269,7 +276,7 @@ describe('plan — fit mode resolution', () => { expect(result.entries[1].fitMode).toBe('contain') }) - it('images inherit global fitMode', () => { + it('images inherit global fitMode when not smart-fit', () => { const cover: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, fitMode: 'cover' } const contain: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, fitMode: 'contain' } const blur: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, fitMode: 'blur-fill' } @@ -278,6 +285,24 @@ describe('plan — fit mode resolution', () => { expect(plan([IMG], contain).entries[0].fitMode).toBe('contain') expect(plan([IMG], blur).entries[0].fitMode).toBe('blur-fill') }) + + it('smart-fit uses blur-fill for landscape media in a portrait canvas', () => { + const settings: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, fitMode: 'smart-fit' } + const result = plan([LANDSCAPE_IMG], settings, undefined, undefined, undefined, undefined, PORTRAIT_CANVAS) + expect(result.entries[0].fitMode).toBe('blur-fill') + }) + + it('smart-fit uses cover for matching landscape media in a landscape canvas', () => { + const settings: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, fitMode: 'smart-fit' } + const result = plan([LANDSCAPE_IMG], settings) + expect(result.entries[0].fitMode).toBe('cover') + }) + + it('smart-fit falls back to contain when media dimensions are unknown', () => { + const settings: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, fitMode: 'smart-fit' } + const result = plan([IMG], settings) + expect(result.entries[0].fitMode).toBe('contain') + }) }) // --- Per-slide overrides --- @@ -326,7 +351,7 @@ describe('plan — per-slide overrides', () => { it('undefined override field falls back to global', () => { const img = { ...makeSlide('a', 'image', 90), overrides: { fitMode: undefined } } const result = plan([img], BASE) - expect(result.entries[0].fitMode).toBe('cover') // global default + expect(result.entries[0].fitMode).toBe('cover') // global default in BASE fixture }) }) @@ -456,7 +481,11 @@ describe('plan — audio clips', () => { }) describe('plan — audio-driven loop golden snapshot', () => { - const CUT_SETTINGS: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, transitionType: 'cut' } + const CUT_SETTINGS: GlobalSettings = { + ...DEFAULT_GLOBAL_SETTINGS, + fitMode: 'cover', + transitionType: 'cut', + } const LONG_AUDIO = [{ blobUrl: 'blob:long', durationInFrames: 250 }] it('matches golden snapshot for looped entries with a partial tail', () => { diff --git a/src/sequence-planner/planner.ts b/src/sequence-planner/planner.ts index fc7c6ba..3fff06d 100644 --- a/src/sequence-planner/planner.ts +++ b/src/sequence-planner/planner.ts @@ -1,5 +1,9 @@ import { resolve } from '../timeline-core/settings' import type { FitMode, GlobalSettings, TransitionType } from '../timeline-core/settings' +import type { AspectRatio } from '../timeline-core/aspect' +import { dimensionsForAspectRatio } from '../timeline-core/aspect' +import { resolveSmartFit } from '../timeline-core/smartFit' +import type { ConcreteFitMode } from '../timeline-core/smartFit' import { isTitleSlide } from '../timeline-core/types' import type { Slide } from '../timeline-core/types' import type { BeatGrid } from '../beat-grid/types' @@ -104,6 +108,7 @@ export function plan( audioClips?: AudioClipInput[], beatGrid?: BeatGrid, concatenatedBeatTimesSecs?: number[], + aspectRatio: AspectRatio = '16:9', ): RenderPlan { if (slides.length === 0) { const audioTotal = totalAudioFrames(audioClips) @@ -146,9 +151,30 @@ export function plan( return raw } - function getFitMode(slide: Slide): FitMode { + const canvas = dimensionsForAspectRatio(aspectRatio) + + function getMediaDimensions(slide: Slide): { height?: number; width?: number } { + if (isTitleSlide(slide)) return {} + const meta = mediaMetadata?.get(slide.filename) + return { + height: slide.height ?? meta?.height, + width: slide.width ?? meta?.width, + } + } + + function resolveConcreteFitMode(slide: Slide, fitMode: FitMode): ConcreteFitMode { + if (fitMode !== 'smart-fit') return fitMode + + const { height, width } = getMediaDimensions(slide) + if (width === undefined || height === undefined) return 'contain' + + return resolveSmartFit(width, height, canvas.width, canvas.height) + } + + function getFitMode(slide: Slide): ConcreteFitMode { if (isTitleSlide(slide)) return 'cover' // unused; TitleSlideView renders its own layout - return slide.type === 'video' ? 'contain' : resolved(slide).fitMode + if (slide.type === 'video') return 'contain' + return resolveConcreteFitMode(slide, resolved(slide).fitMode) } function getTransitionDur(slide: Slide): number { diff --git a/src/sequence-planner/types.ts b/src/sequence-planner/types.ts index 0f82097..4144ec6 100644 --- a/src/sequence-planner/types.ts +++ b/src/sequence-planner/types.ts @@ -1,4 +1,5 @@ -import type { FitMode, TransitionType } from '../timeline-core/settings' +import type { TransitionType } from '../timeline-core/settings' +import type { ConcreteFitMode } from '../timeline-core/smartFit' import type { Slide } from '../timeline-core/types' export type TransitionSpec = { @@ -37,7 +38,7 @@ export type RenderPlanEntry = { startFrame: number durationInFrames: number transitionIn?: TransitionSpec - fitMode: FitMode + fitMode: ConcreteFitMode kenBurns: KenBurnsVector | null videoVolume: number } @@ -65,4 +66,6 @@ export type RenderPlan = { export type MediaMetadata = { durationInFrames?: number + height?: number + width?: number } diff --git a/src/timeline-core/index.ts b/src/timeline-core/index.ts index 641b8a9..71c4362 100644 --- a/src/timeline-core/index.ts +++ b/src/timeline-core/index.ts @@ -16,3 +16,5 @@ export type { } from './settings' export type { AudioClip, MediaSlide, TitleSlide, Slide, MediaType } from './types' export { isTitleSlide } from './types' +export { resolveSmartFit } from './smartFit' +export type { ConcreteFitMode } from './smartFit' diff --git a/src/timeline-core/settings.ts b/src/timeline-core/settings.ts index b07f50b..39dd1c2 100644 --- a/src/timeline-core/settings.ts +++ b/src/timeline-core/settings.ts @@ -2,7 +2,7 @@ import { isTitleSlide } from './types' import type { Slide } from './types' export type TransitionType = 'crossfade' | 'dip-to-black' | 'cut' -export type FitMode = 'cover' | 'contain' | 'blur-fill' +export type FitMode = 'blur-fill' | 'contain' | 'cover' | 'smart-fit' export type KenBurnsMode = 'alternate' | 'zoom-in-only' export type ThemeName = 'classic' | 'energetic' | 'plain' export type Energy = 'calm' | 'medium' | 'punchy' @@ -32,7 +32,7 @@ export const DEFAULT_GLOBAL_SETTINGS: GlobalSettings = { imageDurationSecs: 3, transitionType: 'crossfade', kenBurns: true, - fitMode: 'cover', + fitMode: 'smart-fit', } const FPS = 30 @@ -54,20 +54,20 @@ export const THEMES: Record = { imageDurationSecs: 4, transitionType: 'crossfade', kenBurns: true, - fitMode: 'cover', + fitMode: 'smart-fit', }, energetic: { imageDurationSecs: 2, transitionType: 'cut', kenBurns: true, - fitMode: 'cover', + fitMode: 'smart-fit', kenBurnsMode: 'zoom-in-only', }, plain: { imageDurationSecs: 5, transitionType: 'cut', kenBurns: false, - fitMode: 'cover', + fitMode: 'smart-fit', }, } diff --git a/src/timeline-core/smartFit.test.ts b/src/timeline-core/smartFit.test.ts new file mode 100644 index 0000000..437c23d --- /dev/null +++ b/src/timeline-core/smartFit.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' +import { resolveSmartFit } from './smartFit' + +const LANDSCAPE_16_9 = { height: 1080, width: 1920 } +const PORTRAIT_9_16 = { height: 1920, width: 1080 } +const SQUARE = { height: 1080, width: 1080 } + +describe('resolveSmartFit', () => { + it('uses cover when media and canvas share landscape orientation', () => { + expect(resolveSmartFit( + LANDSCAPE_16_9.width, + LANDSCAPE_16_9.height, + LANDSCAPE_16_9.width, + LANDSCAPE_16_9.height, + )).toBe('cover') + }) + + it('uses cover when media and canvas share portrait orientation', () => { + expect(resolveSmartFit( + PORTRAIT_9_16.width, + PORTRAIT_9_16.height, + PORTRAIT_9_16.width, + PORTRAIT_9_16.height, + )).toBe('cover') + }) + + it('uses blur-fill for landscape media in a portrait canvas', () => { + expect(resolveSmartFit( + LANDSCAPE_16_9.width, + LANDSCAPE_16_9.height, + PORTRAIT_9_16.width, + PORTRAIT_9_16.height, + )).toBe('blur-fill') + }) + + it('uses blur-fill for portrait media in a landscape canvas', () => { + expect(resolveSmartFit( + PORTRAIT_9_16.width, + PORTRAIT_9_16.height, + LANDSCAPE_16_9.width, + LANDSCAPE_16_9.height, + )).toBe('blur-fill') + }) + + it('uses blur-fill for square media in a landscape canvas', () => { + expect(resolveSmartFit( + SQUARE.width, + SQUARE.height, + LANDSCAPE_16_9.width, + LANDSCAPE_16_9.height, + )).toBe('blur-fill') + }) + + it('uses blur-fill for landscape media in a square canvas', () => { + expect(resolveSmartFit( + LANDSCAPE_16_9.width, + LANDSCAPE_16_9.height, + SQUARE.width, + SQUARE.height, + )).toBe('blur-fill') + }) + + it('uses cover for square media in a square canvas', () => { + expect(resolveSmartFit( + SQUARE.width, + SQUARE.height, + SQUARE.width, + SQUARE.height, + )).toBe('cover') + }) + + it('falls back to contain when dimensions are missing or zero', () => { + expect(resolveSmartFit(0, 1080, 1920, 1080)).toBe('contain') + expect(resolveSmartFit(1920, 0, 1920, 1080)).toBe('contain') + }) +}) diff --git a/src/timeline-core/smartFit.ts b/src/timeline-core/smartFit.ts new file mode 100644 index 0000000..4af3cc7 --- /dev/null +++ b/src/timeline-core/smartFit.ts @@ -0,0 +1,46 @@ +import type { FitMode } from './settings' + +export type ConcreteFitMode = Exclude + +const ORIENTATION_THRESHOLD = 0.05 + +function isLandscape(aspect: number): boolean { + return aspect > 1 + ORIENTATION_THRESHOLD +} + +function isPortrait(aspect: number): boolean { + return aspect < 1 - ORIENTATION_THRESHOLD +} + +function isSquare(aspect: number): boolean { + return !isLandscape(aspect) && !isPortrait(aspect) +} + +export function resolveSmartFit( + mediaWidth: number, + mediaHeight: number, + canvasWidth: number, + canvasHeight: number, +): ConcreteFitMode { + if (mediaWidth <= 0 || mediaHeight <= 0 || canvasWidth <= 0 || canvasHeight <= 0) { + return 'contain' + } + + const mediaAspect = mediaWidth / mediaHeight + const canvasAspect = canvasWidth / canvasHeight + + const orientationMismatch = + (isLandscape(mediaAspect) && isPortrait(canvasAspect)) + || (isPortrait(mediaAspect) && isLandscape(canvasAspect)) + + if (orientationMismatch) { + return 'blur-fill' + } + + const shapeMismatch = isSquare(mediaAspect) !== isSquare(canvasAspect) + if (shapeMismatch) { + return 'blur-fill' + } + + return 'cover' +} diff --git a/src/timeline-core/types.ts b/src/timeline-core/types.ts index c7d47ed..d374cc8 100644 --- a/src/timeline-core/types.ts +++ b/src/timeline-core/types.ts @@ -13,8 +13,12 @@ export type MediaSlide = { durationInFrames: number /** excluded from timeline but still in folder */ excluded: boolean + /** intrinsic pixel height when known */ + height?: number /** per-slide setting overrides; absent means "use global defaults" */ overrides?: SlideOverrides + /** intrinsic pixel width when known */ + width?: number } export type TitleSlide = { From 1ec50465d11cbcb6aa6e082d71d58c1a948dc689 Mon Sep 17 00:00:00 2001 From: cbaucom Date: Mon, 22 Jun 2026 16:43:51 -0400 Subject: [PATCH 2/2] Show timeline thumbnails at intrinsic aspect ratio. Use object-contain in timeline blocks and per-slide aspect-ratio on filmstrip cards so portrait and landscape media preview without forced 16:9 cropping. Co-authored-by: Cursor --- specs/smart-fit.md | 6 ++++++ src/editor-shell/StoryboardFilmstrip.tsx | 20 ++++++++++++++------ src/editor-shell/TimelineMediaBlock.tsx | 6 +++--- src/timeline-core/aspect.test.ts | 19 ++++++++++++++++++- src/timeline-core/aspect.ts | 9 +++++++++ src/timeline-core/index.ts | 2 +- 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/specs/smart-fit.md b/specs/smart-fit.md index c430270..9f9a575 100644 --- a/specs/smart-fit.md +++ b/specs/smart-fit.md @@ -24,6 +24,12 @@ Default `cover` fit crops media when its aspect ratio differs from the project c | `project-store/media-loader.ts` | Extract `width`/`height` at import (image bitmap, video track) | | `sequence-planner/planner.ts` | Accept `aspectRatio`, resolve `smart-fit` using slide dims + canvas | | `composition/` | No changes — receives resolved fit modes only | +| Editor thumbnails | `object-contain` + intrinsic `aspect-ratio` from slide dimensions (fallback 16:9) | + +## Thumbnails + +- Timeline blocks: media centered with `object-contain` inside the proportional-width block (full image visible, letterboxed in block). +- Filmstrip cards: container `aspect-ratio` derived from slide `width`/`height`; falls back to 16:9 when unknown. ## Tests diff --git a/src/editor-shell/StoryboardFilmstrip.tsx b/src/editor-shell/StoryboardFilmstrip.tsx index 2f96012..410ac3d 100644 --- a/src/editor-shell/StoryboardFilmstrip.tsx +++ b/src/editor-shell/StoryboardFilmstrip.tsx @@ -1,6 +1,7 @@ import { useRef } from 'react' import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area' import { cn } from '@/lib/utils' +import { mediaAspectRatioCss } from '../timeline-core/aspect' import type { Slide } from '../timeline-core/types' import { isTitleSlide } from '../timeline-core/types' @@ -64,7 +65,14 @@ export function StoryboardFilmstrip({ onDragEnd={() => { dragIndexRef.current = null }} onClick={() => onSlideClick(slide.id)} > -
+
{isTitleSlide(slide) ? (
) : slide.type === 'video' ? (