From 0e0f583a492ea76f979fc7c1f140102e2dedf04e Mon Sep 17 00:00:00 2001 From: npub1223z34hd7vtwc6qj4s7flsxkj644nlre2nthu7lrrmkumhu3xddsrx9r6w <52a228d6edf316ec6812ac3c9fc0d696ab59fc7954d77e7be31eedcddf91335b@sprout-oss.stage.blox.sqprod.co> Date: Wed, 24 Jun 2026 15:42:43 -0700 Subject: [PATCH] feat(media): transcode HEIC/HEIF to JPEG on desktop upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit iPhone HEIC/HEIF photos rendered blank in the desktop composer and were unviewable for everyone because Chromium/the Tauri webview cannot decode HEIC. Mobile already transcodes to JPEG before upload; desktop did not. Add HEIC detection mirroring mobile (full ftyp brand set, broader than the infer crate) and an ffmpeg HEIC->JPEG transcode, wired into both upload paths: process_picked_path (extension OR magic-bytes) and upload_media_bytes (magic-bytes only — no filename on the paste/drag path). Because the composer preview renders the server-returned blob URL, this single Rust fix covers both upload and preview. Missing ffmpeg errors like the video path (no silent blank-preview fallback). Co-authored-by: Taylor Ho Signed-off-by: Taylor Ho --- desktop/src-tauri/src/commands/media.rs | 38 ++- .../src-tauri/src/commands/media_transcode.rs | 255 ++++++++++++++++++ 2 files changed, 292 insertions(+), 1 deletion(-) diff --git a/desktop/src-tauri/src/commands/media.rs b/desktop/src-tauri/src/commands/media.rs index e96e1f36c..6c55613cb 100644 --- a/desktop/src-tauri/src/commands/media.rs +++ b/desktop/src-tauri/src/commands/media.rs @@ -10,7 +10,10 @@ use crate::relay::{ relay_error_message, }; -use super::media_transcode::{is_video_file, transcode_and_extract_poster}; +use super::media_transcode::{ + has_heic_extension, is_heic_file, is_video_file, transcode_and_extract_poster, + transcode_heic_path_to_jpeg_bytes, +}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BlobDescriptor { @@ -305,6 +308,12 @@ async fn process_picked_path( // local attacker from swapping the file between dialog return and read. let mut file = std::fs::File::open(&path).map_err(|e| e.to_string())?; + // Extension hint for HEIC detection — some HEIC files from non-Apple + // tooling carry brands outside HEIC_BRANDS, but the `.heic`/`.heif` + // extension still tells us the webview can't render them. Computed before + // the closure since `path` isn't moved in. + let heic_by_ext = has_heic_extension(&path); + // All sync I/O (sniff, transcode, read) runs off the async runtime to // avoid blocking Tokio worker threads during long ffmpeg transcodes. let (body, poster_bytes) = @@ -326,6 +335,15 @@ async fn process_picked_path( let result = transcode_and_extract_poster(&fd_path); drop(file); // release fd only after ffmpeg is done result + } else if heic_by_ext || is_heic_file(&header[..n]) { + // HEIC/HEIF still: Chromium/the webview can't decode it, so + // transcode to JPEG before upload (mirrors mobile). Resolve the + // fd's real path so ffmpeg reads the pinned inode, and keep + // `file` alive until the transcode finishes. + let fd_path = fd_real_path(&file)?; + let result = transcode_heic_path_to_jpeg_bytes(&fd_path).map(|jpeg| (jpeg, None)); + drop(file); // release fd only after ffmpeg is done + result } else { // Image: read the rest from the already-open fd (TOCTOU-safe). let mut bytes = header[..n].to_vec(); @@ -437,6 +455,24 @@ pub async fn upload_media_bytes( }) .await .map_err(|e| format!("transcode task failed: {e}"))?? + } else if is_heic_file(&data) { + // HEIC/HEIF still pasted/dropped: no filename here, so detection is + // magic-bytes only. ffmpeg needs a path, so write to temp, transcode + // to JPEG, and clean up. (Mirrors mobile's pre-upload transcode.) + tokio::task::spawn_blocking(move || -> Result<(Vec, Option>), String> { + let tmp_input = + std::env::temp_dir().join(format!("buzz-drop-{}", uuid::Uuid::new_v4())); + // Cleanup guard: remove temp file on ALL exit paths (including write failure). + let result = (|| { + std::fs::write(&tmp_input, &data) + .map_err(|e| format!("failed to write temp file: {e}"))?; + transcode_heic_path_to_jpeg_bytes(&tmp_input).map(|jpeg| (jpeg, None)) + })(); + let _ = std::fs::remove_file(&tmp_input); + result + }) + .await + .map_err(|e| format!("transcode task failed: {e}"))?? } else { (data, None) }; diff --git a/desktop/src-tauri/src/commands/media_transcode.rs b/desktop/src-tauri/src/commands/media_transcode.rs index 96aabed6c..f4f96cf6e 100644 --- a/desktop/src-tauri/src/commands/media_transcode.rs +++ b/desktop/src-tauri/src/commands/media_transcode.rs @@ -40,6 +40,60 @@ pub(super) fn is_video_file(buf: &[u8]) -> bool { infer::get(buf).is_some_and(|t| t.mime_type().starts_with("video/")) } +/// HEIC/HEIF compatible-brand codes that mark an ISO-BMFF file as a still +/// HEIF image. Mirrors mobile's `_heicBrands` set in +/// `mobile/lib/shared/relay/media_upload.dart` so detection stays consistent +/// across platforms — deliberately broader than the `infer` crate, which only +/// recognizes `heic`/`heix` majors (or `mif1`/`msf1` with a `heic` compatible +/// brand) and would miss `hevc`/`hevx`/`heim`/`heis`. +const HEIC_BRANDS: &[&[u8; 4]] = &[ + b"heic", b"heix", b"hevc", b"hevx", b"heim", b"heis", b"mif1", b"msf1", +]; + +/// Detect a HEIC/HEIF still image by magic bytes. +/// +/// HEIC/HEIF is an ISO base media file (ISO-BMFF): a `ftyp` box at offset 4 +/// followed by a major brand and a list of compatible brands. We scan the +/// major brand plus the compatible-brand list for any of `HEIC_BRANDS`. +/// +/// Mirrors mobile's `_looksLikeHeicOrHeif`: requires the `ftyp` marker at +/// offset 4 and scans 4-byte brand codes at offsets 8, 12, 16, ... up to the +/// first 32 bytes. The Tauri webview / Chromium cannot decode HEIC, so any +/// match here is transcoded to JPEG before upload. +pub(super) fn is_heic_file(buf: &[u8]) -> bool { + // Need at least the 8-byte box header + 4-byte major brand. + if buf.len() < 12 || &buf[4..8] != b"ftyp" { + return false; + } + + // Scan the major brand (offset 8) and each compatible brand, bounded to + // the first 32 bytes (matches mobile's window). + let upper = buf.len().min(32); + let mut offset = 8; + while offset + 4 <= upper { + let brand: &[u8; 4] = buf[offset..offset + 4].try_into().expect("4-byte slice"); + if HEIC_BRANDS.contains(&brand) { + return true; + } + offset += 4; + } + + false +} + +/// True if a filename ends in `.heic` or `.heif` (case-insensitive). +/// +/// Mirrors mobile's `_hasHeicFileExtension`. Used on the file-picker path as a +/// secondary signal — some HEIC files from non-Apple tooling carry brands not +/// in `HEIC_BRANDS`, but the extension still tells us the webview can't render +/// them. The byte-based path (paste/drag) has no filename and relies solely on +/// `is_heic_file`. +pub(super) fn has_heic_extension(path: &std::path::Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("heic") || ext.eq_ignore_ascii_case("heif")) +} + /// Maximum wall-clock time for an ffmpeg transcode before we kill it. /// 10 minutes is generous for any reasonable video; pathological inputs /// (crafted to cause exponential decode time) get killed instead of @@ -154,6 +208,66 @@ pub(super) fn transcode_to_mp4( Ok(output) } +/// Transcode a HEIC/HEIF still image to JPEG via ffmpeg. +/// +/// The Tauri webview / Chromium cannot decode HEIC, so iPhone photos uploaded +/// as-is render blank in the composer and are unviewable for everyone. This +/// normalizes them to JPEG (the same fix mobile applies before upload). +/// +/// Uses `-frames:v 1` so multi-image HEIF containers (Live Photos, bursts) +/// yield a single still, and `-q:v 2` for high JPEG quality. Returns the path +/// to a temp file. Caller must clean up. +pub(super) fn transcode_heic_to_jpeg( + source: &std::path::Path, + ffmpeg: &std::path::Path, +) -> Result { + // UUID-based temp path — unique across concurrent uploads. + let output = std::env::temp_dir().join(format!("buzz-heic-{}.jpg", uuid::Uuid::new_v4())); + + // Single-frame image decode — 60s is generous even for large HEICs. + let heic_timeout = std::time::Duration::from_secs(60); + + let result = run_ffmpeg_with_timeout( + std::process::Command::new(ffmpeg) + .args(["-y", "-loglevel", "error"]) // suppress progress spam — prevents stderr pipe deadlock + .arg("-i") + .arg(source) // OsStr — handles non-UTF-8 paths on Unix + .args(["-frames:v", "1", "-q:v", "2"]) + .arg(&output) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()), + heic_timeout, + )?; + + if !result.status.success() { + let _ = std::fs::remove_file(&output); + let stderr = String::from_utf8_lossy(&result.stderr); + let detail = stderr + .lines() + .rev() + .find(|l| !l.is_empty() && !l.starts_with(" ")) + .unwrap_or("unknown error"); + return Err(format!("HEIC conversion failed: {detail}")); + } + + Ok(output) +} + +/// Transcode a HEIC/HEIF still image (from a path) to JPEG bytes. +/// +/// Resolves ffmpeg, transcodes, reads the JPEG bytes, and cleans up the temp +/// file. Mirrors `transcode_and_extract_poster` but for images (no poster). +pub(super) fn transcode_heic_path_to_jpeg_bytes( + source: &std::path::Path, +) -> Result, String> { + let ffmpeg_path = find_ffmpeg()?; + let jpeg_path = transcode_heic_to_jpeg(source, &ffmpeg_path)?; + let bytes = + std::fs::read(&jpeg_path).map_err(|e| format!("failed to read transcoded HEIC: {e}")); + let _ = std::fs::remove_file(&jpeg_path); + bytes +} + /// Extract a single JPEG poster frame from a transcoded MP4 via ffmpeg. /// /// Seeks to 1 second (avoids black leader frames), falls back to first frame @@ -280,4 +394,145 @@ mod tests { // It may pass or fail depending on whether ffmpeg is installed. let _ = find_ffmpeg(); } + + /// Build a minimal ISO-BMFF `ftyp` box header with the given major brand + /// and optional compatible brands, suitable for `is_heic_file` testing. + fn ftyp_box(major: &[u8; 4], compatible: &[&[u8; 4]]) -> Vec { + let mut buf = vec![0x00, 0x00, 0x00, 0x00]; // box size (unused by detector) + buf.extend_from_slice(b"ftyp"); + buf.extend_from_slice(major); + buf.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); // minor version + for brand in compatible { + buf.extend_from_slice(*brand); + } + buf + } + + #[test] + fn test_is_heic_file_major_brands() { + // Every brand in HEIC_BRANDS should be detected as the major brand. + for brand in HEIC_BRANDS { + let buf = ftyp_box(brand, &[]); + assert!(is_heic_file(&buf), "major brand {brand:?} not detected"); + } + } + + #[test] + fn test_is_heic_file_variants_infer_misses() { + // These brands are detected by mobile but NOT by the `infer` crate's + // HEIC heuristic — the whole reason we mirror mobile's full set. + for brand in [b"hevc", b"hevx", b"heim", b"heis"] { + let buf = ftyp_box(brand, &[]); + assert!(is_heic_file(&buf), "variant brand {brand:?} not detected"); + } + } + + #[test] + fn test_is_heic_file_compatible_brand() { + // Major brand is generic (mif1), HEIC signaled via compatible brand. + let buf = ftyp_box(b"mif1", &[b"heic"]); + assert!(is_heic_file(&buf)); + } + + #[test] + fn test_is_heic_file_jpeg_is_not_heic() { + let jpeg = [ + 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, b'J', b'F', b'I', b'F', 0x00, 0x01, + ]; + assert!(!is_heic_file(&jpeg)); + } + + #[test] + fn test_is_heic_file_mp4_is_not_heic() { + // An MP4 ftyp box (isom) must not be misdetected as HEIC. + let mp4 = ftyp_box(b"isom", &[b"isom", b"iso2"]); + assert!(!is_heic_file(&mp4)); + } + + #[test] + fn test_is_heic_file_empty() { + assert!(!is_heic_file(&[])); + } + + #[test] + fn test_is_heic_file_too_short() { + // Has `ftyp` marker but fewer than 12 bytes — below mobile's threshold. + let buf = [0x00, 0x00, 0x00, 0x00, b'f', b't', b'y', b'p']; + assert!(!is_heic_file(&buf)); + } + + #[test] + fn test_is_heic_file_no_ftyp_marker() { + // 12+ bytes containing a HEIC brand but no `ftyp` at offset 4. + let mut buf = vec![0u8; 16]; + buf[8..12].copy_from_slice(b"heic"); + assert!(!is_heic_file(&buf)); + } + + #[test] + fn test_is_heic_file_brand_past_window() { + // A HEIC brand sitting beyond the 32-byte scan window must not match, + // matching mobile's bounded scan. Use non-HEIC major + filler brands + // so the only HEIC brand present is the one pushed past offset 32. + let mut buf = ftyp_box(b"isom", &[b"iso2", b"iso4", b"avc1", b"mp41", b"mp42"]); + buf.extend_from_slice(b"heic"); // lands at offset 36, past the window + assert!(!is_heic_file(&buf)); + } + + #[test] + fn test_has_heic_extension() { + use std::path::Path; + assert!(has_heic_extension(Path::new("IMG_1234.HEIC"))); + assert!(has_heic_extension(Path::new("photo.heic"))); + assert!(has_heic_extension(Path::new("photo.heif"))); + assert!(has_heic_extension(Path::new("photo.HEIF"))); + assert!(!has_heic_extension(Path::new("photo.jpg"))); + assert!(!has_heic_extension(Path::new("photo.png"))); + assert!(!has_heic_extension(Path::new("noextension"))); + } + + /// Round-trip transcode test, gated on ffmpeg being present so CI without + /// ffmpeg doesn't fail. Generates a HEIC via ffmpeg, then transcodes it + /// back to JPEG and asserts the output is a valid JPEG. + #[test] + fn test_transcode_heic_round_trip() { + let Ok(ffmpeg) = find_ffmpeg() else { + eprintln!("skipping HEIC round-trip: ffmpeg not found"); + return; + }; + + // Generate a small HEIC test image from a synthetic color source. + let heic_path = + std::env::temp_dir().join(format!("buzz-test-{}.heic", uuid::Uuid::new_v4())); + let gen = std::process::Command::new(&ffmpeg) + .args(["-y", "-loglevel", "error", "-f", "lavfi", "-i"]) + .arg("color=c=red:s=64x64:d=1") + .args(["-frames:v", "1"]) + .arg(&heic_path) + .output(); + + let gen = match gen { + Ok(o) if o.status.success() && heic_path.exists() => o, + other => { + // This ffmpeg build can't encode HEIC — skip rather than fail. + eprintln!("skipping HEIC round-trip: ffmpeg cannot encode HEIC: {other:?}"); + let _ = std::fs::remove_file(&heic_path); + return; + } + }; + drop(gen); + + // Sanity: the generated file should be detected as HEIC. + let heic_bytes = std::fs::read(&heic_path).expect("read generated heic"); + assert!( + is_heic_file(&heic_bytes), + "generated file not detected as HEIC" + ); + + // Transcode to JPEG bytes and verify the JPEG magic. + let jpeg = transcode_heic_path_to_jpeg_bytes(&heic_path).expect("transcode to jpeg"); + let _ = std::fs::remove_file(&heic_path); + assert!(jpeg.len() > 2, "empty jpeg output"); + assert_eq!(&jpeg[0..2], &[0xFF, 0xD8], "output is not a JPEG"); + } }