From f622fe16eadb05fb21409bb4b8cc846628ec446e Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 16 May 2026 17:40:11 -0400 Subject: [PATCH 001/271] Optimize Metal HTJ2K resident encode pipeline --- .../benches/encode_stages.rs | 128 +- crates/signinum-j2k-metal/src/compute.rs | 224 +++- crates/signinum-j2k-metal/src/encode.rs | 1048 ++++++++++++----- crates/signinum-j2k-metal/src/lib.rs | 9 +- crates/signinum-j2k-metal/src/mct.metal | 54 + 5 files changed, 1141 insertions(+), 322 deletions(-) diff --git a/crates/signinum-j2k-metal/benches/encode_stages.rs b/crates/signinum-j2k-metal/benches/encode_stages.rs index 3e664944..db57ff5e 100644 --- a/crates/signinum-j2k-metal/benches/encode_stages.rs +++ b/crates/signinum-j2k-metal/benches/encode_stages.rs @@ -1,18 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; #[cfg(target_os = "macos")] -use signinum_core::PixelFormat; +use signinum_core::{DeviceSubmission, PixelFormat}; use signinum_j2k::{ encode_j2k_lossless, EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, - J2kLosslessEncodeOptions, J2kLosslessSamples, + J2kLosslessEncodeOptions, J2kLosslessSamples, J2kProgressionOrder, }; use signinum_j2k_metal::MetalEncodeStageAccelerator; #[cfg(target_os = "macos")] use signinum_j2k_metal::{ - encode_lossless_from_padded_metal_buffer_with_report, MetalBackendSession, - MetalLosslessEncodeTile, + encode_lossless_from_padded_metal_buffer_with_report, + submit_lossless_from_padded_metal_buffers_to_metal_batch, MetalBackendSession, + MetalLosslessEncodeConfig, MetalLosslessEncodeTile, }; +use signinum_j2k_native::J2kHtCodeBlockEncodeJob; use signinum_j2k_native::{J2kEncodeStageAccelerator, J2kForwardDwt53Job, J2kForwardRctJob}; const BENCH_DIMS: &[u32] = &[512, 1024, 2048]; @@ -203,6 +205,105 @@ fn bench_encode_stages(c: &mut Criterion) { } } encode.finish(); + + let mut ht_tier1 = c.benchmark_group("j2k_metal_ht_tier1_code_blocks"); + for &count in &[192usize, 768] { + let blocks = generate_ht_code_block_coefficients(count, 64, 64); + ht_tier1.bench_with_input(BenchmarkId::new("cpu", count), &blocks, |b, blocks| { + b.iter(|| { + blocks + .iter() + .map(|coefficients| { + signinum_j2k_native::encode_ht_code_block_scalar( + black_box(coefficients), + 64, + 64, + 10, + ) + .expect("CPU HTJ2K code-block encode") + }) + .collect::>() + }); + }); + + if metal_encode_available() { + ht_tier1.bench_with_input(BenchmarkId::new("metal", count), &blocks, |b, blocks| { + let jobs = blocks + .iter() + .map(|coefficients| J2kHtCodeBlockEncodeJob { + coefficients, + width: 64, + height: 64, + total_bitplanes: 10, + }) + .collect::>(); + b.iter(|| { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let encoded = accelerator + .encode_ht_code_blocks(black_box(&jobs)) + .expect("Metal HTJ2K Tier-1 batch") + .expect("Metal HTJ2K Tier-1 dispatch"); + assert_eq!(encoded.len(), jobs.len()); + encoded + }); + }); + } + } + ht_tier1.finish(); + + #[cfg(target_os = "macos")] + if metal_encode_available() { + let mut htj2k_batch = c.benchmark_group("j2k_metal_htj2k_rpcl_rgb8_512_batch"); + let session = MetalBackendSession::system_default().expect("Metal session"); + let pixels = generate_rgb8_pixels(512, 512); + let buffer = private_buffer_with_bytes(&session, &pixels); + let options = J2kLosslessEncodeOptions { + backend: EncodeBackendPreference::RequireDevice, + validation: J2kEncodeValidation::External, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + progression: J2kProgressionOrder::Rpcl, + ..J2kLosslessEncodeOptions::default() + }; + let config = MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: None, + gpu_encode_memory_budget_bytes: None, + }; + for &count in &[16usize, 64, 128] { + htj2k_batch.bench_with_input( + BenchmarkId::new("resident_metal", count), + &count, + |b, &count| { + b.iter(|| { + let requests = (0..count) + .map(|_| MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 512, + height: 512, + pitch_bytes: 512 * 3, + output_width: 512, + output_height: 512, + format: PixelFormat::Rgb8, + }) + .collect::>(); + let submitted = submit_lossless_from_padded_metal_buffers_to_metal_batch( + black_box(&requests), + &options, + &session, + config, + ) + .expect("submit resident Metal HTJ2K RPCL batch"); + let outcome = submitted + .wait() + .expect("wait resident Metal HTJ2K RPCL batch"); + assert_eq!(outcome.outcomes.len(), count); + black_box(outcome.stats) + }); + }, + ); + } + htj2k_batch.finish(); + } } fn metal_encode_available() -> bool { @@ -259,6 +360,23 @@ fn generate_rgb8_pixels(width: u32, height: u32) -> Vec { pixels } +fn generate_ht_code_block_coefficients(count: usize, width: usize, height: usize) -> Vec> { + (0..count) + .map(|block| { + (0..width * height) + .map(|idx| { + let raw = ((idx * 37 + block * 19 + idx / 11) & 0x3ff) as i32 - 512; + if (idx + block) % 23 == 0 || idx % 41 == 0 { + 0 + } else { + raw + } + }) + .collect() + }) + .collect() +} + #[cfg(target_os = "macos")] fn private_buffer_with_bytes(session: &MetalBackendSession, bytes: &[u8]) -> metal::Buffer { let upload = session.device().new_buffer_with_data( diff --git a/crates/signinum-j2k-metal/src/compute.rs b/crates/signinum-j2k-metal/src/compute.rs index 0a905078..6a83d70d 100644 --- a/crates/signinum-j2k-metal/src/compute.rs +++ b/crates/signinum-j2k-metal/src/compute.rs @@ -10,7 +10,7 @@ use std::{ collections::HashMap, mem::{size_of, size_of_val}, sync::{Arc, Mutex, OnceLock}, - time::Duration, + time::{Duration, Instant}, }; #[cfg(target_os = "macos")] @@ -54,6 +54,7 @@ static HT_BATCH_COEFFICIENT_COPY_BLITS: AtomicUsize = AtomicUsize::new(0); #[cfg(all(target_os = "macos", test))] std::thread_local! { static PRIVATE_BUFFER_POOL_MISSES: Cell = const { Cell::new(0) }; + static LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES: Cell = const { Cell::new(0) }; static HT_SIMD_PROTOTYPE_DISPATCHES: Cell = const { Cell::new(0) }; static HT_SIMD_PROTOTYPE_ROUTE_OVERRIDE: Cell> = const { Cell::new(None) }; } @@ -78,6 +79,16 @@ pub(crate) fn private_buffer_pool_misses_for_test() -> usize { PRIVATE_BUFFER_POOL_MISSES.with(Cell::get) } +#[cfg(all(target_os = "macos", test))] +pub(crate) fn reset_lossless_deinterleave_rct_fused_dispatches_for_test() { + LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES.with(|dispatches| dispatches.set(0)); +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn lossless_deinterleave_rct_fused_dispatches_for_test() -> usize { + LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES.with(Cell::get) +} + #[cfg(all(target_os = "macos", test))] pub(crate) fn reset_ht_simd_prototype_dispatches_for_test() { HT_SIMD_PROTOTYPE_DISPATCHES.with(|dispatches| dispatches.set(0)); @@ -1202,8 +1213,9 @@ fn ht_simd_prototype_env_requested() -> bool { } #[cfg(target_os = "macos")] -fn metal_profile_stages_enabled() -> bool { - env_flag_enabled(METAL_PROFILE_STAGES_ENV) +pub(crate) fn metal_profile_stages_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_STAGES_ENV)) } #[cfg(target_os = "macos")] @@ -1521,6 +1533,7 @@ struct MetalRuntime { validate_bytes_equal: ComputePipelineState, copy_interleaved_padded: ComputePipelineState, lossless_deinterleave_to_planes: ComputePipelineState, + lossless_deinterleave_rct_rgb8_to_planes: ComputePipelineState, lossless_extract_coefficients: ComputePipelineState, pack_gray8: ComputePipelineState, pack_rgb8: ComputePipelineState, @@ -1724,6 +1737,9 @@ impl MetalRuntime { validate_bytes_equal: pipeline("j2k_validate_bytes_equal")?, copy_interleaved_padded: pipeline("j2k_copy_interleaved_padded")?, lossless_deinterleave_to_planes: pipeline("j2k_lossless_deinterleave_to_planes")?, + lossless_deinterleave_rct_rgb8_to_planes: pipeline( + "j2k_lossless_deinterleave_rct_rgb8_to_planes", + )?, lossless_extract_coefficients: pipeline("j2k_lossless_extract_coefficients")?, pack_gray8: pipeline("j2k_pack_gray8")?, pack_rgb8: pipeline("j2k_pack_rgb8")?, @@ -6621,6 +6637,21 @@ pub(crate) struct J2kResidentClassicBatchEncodeItem { pub(crate) codestream: J2kLosslessCodestreamAssemblyJob, } +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct J2kResidentEncodeStageStats { + pub(crate) ht_table_build_duration: Duration, + pub(crate) ht_buffer_allocation_duration: Duration, + pub(crate) ht_command_encode_duration: Duration, + pub(crate) code_block_count: usize, +} + +#[cfg(target_os = "macos")] +pub(crate) struct J2kResidentLosslessCodestreamBatchResult { + pub(crate) codestreams: Vec, + pub(crate) stage_stats: J2kResidentEncodeStageStats, +} + #[cfg(target_os = "macos")] pub(crate) struct J2kPendingResidentLosslessCodestreamBatch { device: Device, @@ -6632,6 +6663,7 @@ pub(crate) struct J2kPendingResidentLosslessCodestreamBatch { retained_command_buffers: Vec, _retained_buffers: Vec, recyclable_private_buffers: Vec<(usize, Buffer)>, + stage_stats: J2kResidentEncodeStageStats, status_stage: &'static str, length_error: &'static str, capacity_error: &'static str, @@ -6702,12 +6734,13 @@ pub(crate) fn wait_resident_lossless_codestream( #[cfg(target_os = "macos")] pub(crate) fn wait_resident_lossless_codestream_batch( pending: J2kPendingResidentLosslessCodestreamBatch, -) -> Result, Error> { +) -> Result { pending.command_buffer.wait_until_completed(); let gpu_duration = completed_command_buffers_gpu_duration( &pending.retained_command_buffers, &pending.command_buffer, ); + let stage_stats = pending.stage_stats; let recyclable_private_buffers = pending.recyclable_private_buffers; with_runtime_for_device(&pending.device, |runtime| { recycle_private_buffers(runtime, recyclable_private_buffers); @@ -6750,7 +6783,10 @@ pub(crate) fn wait_resident_lossless_codestream_batch( gpu_duration: gpu_duration_share, }); } - Ok(codestreams) + Ok(J2kResidentLosslessCodestreamBatchResult { + codestreams, + stage_stats, + }) } #[cfg(target_os = "macos")] @@ -6864,6 +6900,87 @@ fn dispatch_lossless_deinterleave( Ok(()) } +#[cfg(target_os = "macos")] +fn dispatch_lossless_deinterleave_rct_rgb8( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + job: J2kLosslessDevicePrepareJob<'_>, + plane0: &Buffer, + plane1: &Buffer, + plane2: &Buffer, + status_buffer: &Buffer, +) -> Result<(), Error> { + let input_byte_offset = + u64::try_from(job.input_byte_offset).map_err(|_| Error::MetalKernel { + message: "J2K Metal resident encode input offset exceeds u64".to_string(), + })?; + let src_stride = u32::try_from(job.input_pitch_bytes).map_err(|_| Error::MetalKernel { + message: "J2K Metal resident encode input pitch exceeds u32".to_string(), + })?; + let sample_offset = if job.bit_depth == 0 || job.bit_depth > 16 { + return Err(Error::MetalKernel { + message: "J2K Metal resident encode bit depth must be 1-16".to_string(), + }); + } else { + 1u32 << (u32::from(job.bit_depth) - 1) + }; + let params = J2kLosslessDeinterleaveParams { + src_width: job.input_width, + src_height: job.input_height, + src_stride, + dst_width: job.output_width, + dst_height: job.output_height, + components: u32::from(job.components), + bytes_per_sample: u32::from(job.bytes_per_sample), + sample_offset, + }; + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K input deinterleave + RCT"); + encoder.set_compute_pipeline_state(&runtime.lossless_deinterleave_rct_rgb8_to_planes); + encoder.set_buffer(0, Some(job.input), input_byte_offset); + encoder.set_buffer(1, Some(plane0), 0); + encoder.set_buffer(2, Some(plane1), 0); + encoder.set_buffer(3, Some(plane2), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.set_buffer(5, Some(status_buffer), 0); + let width = runtime + .lossless_deinterleave_rct_rgb8_to_planes + .thread_execution_width() + .max(1); + let max_threads = runtime + .lossless_deinterleave_rct_rgb8_to_planes + .max_total_threads_per_threadgroup() + .max(width); + let height = (max_threads / width).max(1); + encoder.dispatch_threads( + MTLSize { + width: u64::from(job.output_width), + height: u64::from(job.output_height), + depth: 1, + }, + MTLSize { + width, + height, + depth: 1, + }, + ); + encoder.end_encoding(); + #[cfg(test)] + LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES.with(|dispatches| { + dispatches.set(dispatches.get().saturating_add(1)); + }); + Ok(()) +} + +#[cfg(target_os = "macos")] +fn lossless_deinterleave_rct_rgb8_supported(job: J2kLosslessDevicePrepareJob<'_>) -> bool { + job.components == 3 && job.bytes_per_sample == 1 +} + #[cfg(target_os = "macos")] fn dispatch_forward_rct_on_buffers( runtime: &MetalRuntime, @@ -7066,11 +7183,6 @@ fn lossless_prepare_sizes( reason: "J2K Metal resident encode supports 8-bit or 16-bit samples", }); } - if job.num_decomposition_levels > 1 { - return Err(Error::UnsupportedMetalRequest { - reason: "J2K Metal resident encode currently supports up to one DWT level", - }); - } let plane_len = (job.output_width as usize) .checked_mul(job.output_height as usize) .ok_or_else(|| Error::MetalKernel { @@ -7131,15 +7243,27 @@ pub(crate) fn prepare_lossless_device_code_blocks( ); let command_buffer = runtime.queue.new_command_buffer(); - dispatch_lossless_deinterleave( - runtime, - command_buffer, - job, - &plane_buffers[0], - &plane_buffers[1], - &plane_buffers[2], - )?; - if job.components == 3 { + if lossless_deinterleave_rct_rgb8_supported(job) { + dispatch_lossless_deinterleave_rct_rgb8( + runtime, + command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + &status_buffer, + )?; + } else { + dispatch_lossless_deinterleave( + runtime, + command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + )?; + } + if job.components == 3 && !lossless_deinterleave_rct_rgb8_supported(job) { dispatch_forward_rct_on_buffers( runtime, command_buffer, @@ -7281,21 +7405,33 @@ pub(crate) fn prepare_lossless_device_code_blocks_batch( MTLResourceOptions::StorageModeShared, ); - dispatch_lossless_deinterleave( - runtime, - command_buffer, - job, - &plane_buffers[0], - &plane_buffers[1], - &plane_buffers[2], - ) + if lossless_deinterleave_rct_rgb8_supported(job) { + dispatch_lossless_deinterleave_rct_rgb8( + runtime, + command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + &status_buffer, + ) + } else { + dispatch_lossless_deinterleave( + runtime, + command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + ) + } .map_err(|err| Error::MetalKernel { message: format!( "J2K Metal resident batch coefficient prep failed at tile {}: {err}", item.tile_index ), })?; - if job.components == 3 { + if job.components == 3 && !lossless_deinterleave_rct_rgb8_supported(job) { dispatch_forward_rct_on_buffers( runtime, command_buffer, @@ -12660,6 +12796,11 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( } with_runtime_for_device(&session.device, |runtime| { + let profile_stages = metal_profile_stages_enabled(); + let mut stage_stats = J2kResidentEncodeStageStats::default(); + let mut ht_table_build_duration = Duration::ZERO; + let mut ht_command_encode_duration = Duration::ZERO; + let mut ht_table_build_started = profile_stages.then(Instant::now); let command_buffer = runtime.queue.new_command_buffer(); label_command_buffer(command_buffer, "signinum-j2k htj2k resident encode batch"); let mut retained_command_buffers = Vec::with_capacity(prepared_tiles.len()); @@ -12784,8 +12925,12 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( let tier1_job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { message: "HTJ2K Metal batch Tier-1 job count exceeds u32".to_string(), })?; + if let Some(started) = ht_table_build_started.take() { + ht_table_build_duration = ht_table_build_duration.saturating_add(started.elapsed()); + } let kernel = HtEncodeCodeBlocksKernel::from_env(runtime); if tier1_job_count > 0 { + let command_encode_started = profile_stages.then(Instant::now); let encoder = command_buffer.new_compute_command_encoder(); label_compute_encoder(encoder, "HTJ2K Tier-1 encode"); let pipeline = kernel.pipeline(runtime)?; @@ -12804,8 +12949,13 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( ); kernel.dispatch(encoder, pipeline, tier1_job_count); encoder.end_encoding(); + if let Some(started) = command_encode_started { + ht_command_encode_duration = + ht_command_encode_duration.saturating_add(started.elapsed()); + } } + ht_table_build_started = profile_stages.then(Instant::now); let mut packet_resolutions = Vec::::new(); let mut packet_subbands = Vec::::new(); let mut resident_blocks = Vec::::new(); @@ -13180,6 +13330,10 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( })?; } + if let Some(started) = ht_table_build_started.take() { + ht_table_build_duration = ht_table_build_duration.saturating_add(started.elapsed()); + } + let ht_buffer_allocation_started = profile_stages.then(Instant::now); let packet_resolution_buffer = copied_slice_buffer(&runtime.device, &packet_resolutions); let packet_subband_buffer = copied_slice_buffer(&runtime.device, &packet_subbands); let resident_block_buffer = copied_slice_buffer(&runtime.device, &resident_blocks); @@ -13220,7 +13374,11 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( (assembly_jobs.len() * size_of::()) as u64, MTLResourceOptions::StorageModeShared, ); + if let Some(started) = ht_buffer_allocation_started { + stage_stats.ht_buffer_allocation_duration = started.elapsed(); + } + let command_encode_started = profile_stages.then(Instant::now); let resident_block_params = J2kResidentPacketBlockParams { block_count: u32::try_from(resident_blocks.len()).map_err(|_| Error::MetalKernel { message: "HTJ2K Metal batch resident block count exceeds u32".to_string(), @@ -13317,6 +13475,10 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( ); encoder.end_encoding(); command_buffer.commit(); + if let Some(started) = command_encode_started { + ht_command_encode_duration = + ht_command_encode_duration.saturating_add(started.elapsed()); + } for tile in prepared_tiles { retained_command_buffers.push(tile.prepare_command_buffer); @@ -13344,6 +13506,10 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( retained_buffers.push(packet_status_buffer); retained_buffers.push(codestream_job_buffer); + stage_stats.ht_table_build_duration = ht_table_build_duration; + stage_stats.ht_command_encode_duration = ht_command_encode_duration; + stage_stats.code_block_count = tier1_jobs.len(); + Ok(J2kPendingResidentLosslessCodestreamBatch { device: runtime.device.clone(), buffer: codestream_buffer, @@ -13354,6 +13520,7 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( retained_command_buffers, _retained_buffers: retained_buffers, recyclable_private_buffers, + stage_stats, status_stage: "HTJ2K batched codestream assembly", length_error: "HTJ2K Metal batched codestream output length exceeds usize", capacity_error: "HTJ2K Metal batched codestream output length exceeds buffer", @@ -14125,6 +14292,7 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( retained_command_buffers, _retained_buffers: retained_buffers, recyclable_private_buffers, + stage_stats: J2kResidentEncodeStageStats::default(), status_stage: "J2K batched codestream assembly", length_error: "J2K Metal batched codestream output length exceeds usize", capacity_error: "J2K Metal batched codestream output length exceeds buffer", diff --git a/crates/signinum-j2k-metal/src/encode.rs b/crates/signinum-j2k-metal/src/encode.rs index 7c3f1ffb..761fd316 100644 --- a/crates/signinum-j2k-metal/src/encode.rs +++ b/crates/signinum-j2k-metal/src/encode.rs @@ -547,6 +547,66 @@ pub struct MetalLosslessEncodeConfig { pub gpu_encode_memory_budget_bytes: Option, } +/// Optional resident Metal encode stage timings. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MetalLosslessEncodeStageStats { + pub plan_duration: Duration, + pub prepare_submit_duration: Duration, + pub ht_table_build_duration: Duration, + pub ht_buffer_allocation_duration: Duration, + pub ht_command_encode_duration: Duration, + pub codestream_wait_duration: Duration, + pub chunk_count: usize, + pub tile_count: usize, + pub code_block_count: usize, +} + +impl MetalLosslessEncodeStageStats { + pub fn has_timings(&self) -> bool { + self.plan_duration > Duration::ZERO + || self.prepare_submit_duration > Duration::ZERO + || self.ht_table_build_duration > Duration::ZERO + || self.ht_buffer_allocation_duration > Duration::ZERO + || self.ht_command_encode_duration > Duration::ZERO + || self.codestream_wait_duration > Duration::ZERO + } + + fn add_assign(&mut self, other: Self) { + self.plan_duration = self.plan_duration.saturating_add(other.plan_duration); + self.prepare_submit_duration = self + .prepare_submit_duration + .saturating_add(other.prepare_submit_duration); + self.ht_table_build_duration = self + .ht_table_build_duration + .saturating_add(other.ht_table_build_duration); + self.ht_buffer_allocation_duration = self + .ht_buffer_allocation_duration + .saturating_add(other.ht_buffer_allocation_duration); + self.ht_command_encode_duration = self + .ht_command_encode_duration + .saturating_add(other.ht_command_encode_duration); + self.codestream_wait_duration = self + .codestream_wait_duration + .saturating_add(other.codestream_wait_duration); + self.chunk_count = self.chunk_count.saturating_add(other.chunk_count); + self.tile_count = self.tile_count.saturating_add(other.tile_count); + self.code_block_count = self.code_block_count.saturating_add(other.code_block_count); + } +} + +#[cfg(target_os = "macos")] +impl From for MetalLosslessEncodeStageStats { + fn from(stats: compute::J2kResidentEncodeStageStats) -> Self { + Self { + ht_table_build_duration: stats.ht_table_build_duration, + ht_buffer_allocation_duration: stats.ht_buffer_allocation_duration, + ht_command_encode_duration: stats.ht_command_encode_duration, + code_block_count: stats.code_block_count, + ..Self::default() + } + } +} + /// Resolved resident Metal lossless J2K/HTJ2K tile batch encode metrics. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct MetalLosslessEncodeBatchStats { @@ -557,6 +617,7 @@ pub struct MetalLosslessEncodeBatchStats { pub estimated_peak_bytes_per_tile: usize, pub max_observed_inflight_tiles: usize, pub encode_wall_duration: Duration, + pub stage_stats: MetalLosslessEncodeStageStats, } /// Resident Metal lossless J2K/HTJ2K tile batch output and batch-level metrics. @@ -575,6 +636,11 @@ pub struct SubmittedJ2kLosslessMetalEncodeBatch { state: SubmittedJ2kLosslessMetalEncodeBatchState, } +#[cfg(target_os = "macos")] +pub struct SubmittedJ2kLosslessMetalBufferEncodeBatch { + state: SubmittedJ2kLosslessMetalBufferEncodeBatchState, +} + #[cfg(target_os = "macos")] enum SubmittedJ2kLosslessMetalEncodeBatchState { Ready(Vec), @@ -587,6 +653,17 @@ enum SubmittedJ2kLosslessMetalEncodeBatchState { }, } +#[cfg(target_os = "macos")] +enum SubmittedJ2kLosslessMetalBufferEncodeBatchState { + Resident(Box), + Deferred { + tiles: Vec, + options: J2kLosslessEncodeOptions, + session: crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + }, +} + #[cfg(target_os = "macos")] struct OwnedMetalLosslessEncodeTile { buffer: Buffer, @@ -638,6 +715,11 @@ pub struct SubmittedJ2kLosslessMetalEncodeBatch { _private: (), } +#[cfg(not(target_os = "macos"))] +pub struct SubmittedJ2kLosslessMetalBufferEncodeBatch { + _private: (), +} + #[cfg(target_os = "macos")] impl DeviceSubmission for SubmittedJ2kLosslessMetalEncode { type Output = EncodedJ2k; @@ -682,6 +764,28 @@ impl DeviceSubmission for SubmittedJ2kLosslessMetalEncodeBatch { } } +#[cfg(target_os = "macos")] +impl DeviceSubmission for SubmittedJ2kLosslessMetalBufferEncodeBatch { + type Output = MetalLosslessBufferEncodeBatchOutcome; + type Error = crate::Error; + + fn wait(self) -> Result { + match self.state { + SubmittedJ2kLosslessMetalBufferEncodeBatchState::Resident(submitted) => { + wait_submitted_resident_lossless_buffer_encode_batch(*submitted) + } + SubmittedJ2kLosslessMetalBufferEncodeBatchState::Deferred { + tiles, + options, + session, + staging, + } => encode_owned_lossless_tiles_to_metal_buffer_fallback_batch( + &tiles, options, &session, staging, + ), + } + } +} + #[cfg(not(target_os = "macos"))] impl DeviceSubmission for SubmittedJ2kLosslessMetalEncode { type Output = EncodedJ2k; @@ -704,6 +808,17 @@ impl DeviceSubmission for SubmittedJ2kLosslessMetalEncodeBatch { } } +#[cfg(not(target_os = "macos"))] +impl DeviceSubmission for SubmittedJ2kLosslessMetalBufferEncodeBatch { + type Output = MetalLosslessBufferEncodeBatchOutcome; + type Error = crate::Error; + + fn wait(self) -> Result { + let _ = self; + Err(crate::Error::MetalUnavailable) + } +} + #[cfg(target_os = "macos")] pub fn encode_lossless_from_metal_buffer( tile: MetalLosslessEncodeTile<'_>, @@ -912,13 +1027,13 @@ pub fn encode_lossless_from_metal_buffers_to_metal_with_report( } #[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffers_to_metal_batch( +pub fn submit_lossless_from_metal_buffers_to_metal_batch( tiles: &[MetalLosslessEncodeTile<'_>], options: &J2kLosslessEncodeOptions, session: &crate::MetalBackendSession, config: MetalLosslessEncodeConfig, -) -> Result { - encode_lossless_tiles_to_metal_buffer_batch( +) -> Result { + submit_lossless_tiles_to_metal_buffer_batch( tiles, *options, session, @@ -927,6 +1042,16 @@ pub fn encode_lossless_from_metal_buffers_to_metal_batch( ) } +#[cfg(target_os = "macos")] +pub fn encode_lossless_from_metal_buffers_to_metal_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + config: MetalLosslessEncodeConfig, +) -> Result { + submit_lossless_from_metal_buffers_to_metal_batch(tiles, options, session, config)?.wait() +} + #[cfg(target_os = "macos")] pub fn encode_lossless_from_padded_metal_buffers( tiles: &[MetalLosslessEncodeTile<'_>], @@ -1010,13 +1135,13 @@ pub fn encode_lossless_from_padded_metal_buffers_to_metal_with_report( } #[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffers_to_metal_batch( +pub fn submit_lossless_from_padded_metal_buffers_to_metal_batch( tiles: &[MetalLosslessEncodeTile<'_>], options: &J2kLosslessEncodeOptions, session: &crate::MetalBackendSession, config: MetalLosslessEncodeConfig, -) -> Result { - encode_lossless_tiles_to_metal_buffer_batch( +) -> Result { + submit_lossless_tiles_to_metal_buffer_batch( tiles, *options, session, @@ -1025,6 +1150,17 @@ pub fn encode_lossless_from_padded_metal_buffers_to_metal_batch( ) } +#[cfg(target_os = "macos")] +pub fn encode_lossless_from_padded_metal_buffers_to_metal_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + config: MetalLosslessEncodeConfig, +) -> Result { + submit_lossless_from_padded_metal_buffers_to_metal_batch(tiles, options, session, config)? + .wait() +} + #[cfg(target_os = "macos")] fn encode_lossless_tiles_with_report( tiles: &[MetalLosslessEncodeTile<'_>], @@ -1113,25 +1249,58 @@ fn encode_lossless_owned_tiles_with_report( } #[cfg(target_os = "macos")] -fn encode_lossless_tiles_to_metal_buffer_batch( +fn submit_lossless_tiles_to_metal_buffer_batch( tiles: &[MetalLosslessEncodeTile<'_>], options: J2kLosslessEncodeOptions, session: &crate::MetalBackendSession, staging: MetalEncodeInputStaging, config: MetalLosslessEncodeConfig, -) -> Result { +) -> Result { if options.backend != EncodeBackendPreference::CpuOnly { - if let Some(batch) = try_encode_resident_lossless_tiles_to_metal_buffer_batch( + if let Some(submitted) = try_submit_resident_lossless_tiles_to_metal_buffer_batch( tiles, options, session, staging, config, )? { - return Ok(batch); + return Ok(SubmittedJ2kLosslessMetalBufferEncodeBatch { + state: SubmittedJ2kLosslessMetalBufferEncodeBatchState::Resident(Box::new( + submitted, + )), + }); } } - let mut outcomes = Vec::with_capacity(tiles.len()); + let mut owned = Vec::with_capacity(tiles.len()); for &tile in tiles { + validate_metal_encode_tile(tile)?; + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + lossless_sample_shape(tile.format)?; + validate_padded_contiguous_metal_encode_tile(tile, tile.format.bytes_per_pixel())?; + } + owned.push(OwnedMetalLosslessEncodeTile::from_tile(tile)); + } + Ok(SubmittedJ2kLosslessMetalBufferEncodeBatch { + state: SubmittedJ2kLosslessMetalBufferEncodeBatchState::Deferred { + tiles: owned, + options, + session: session.clone(), + staging, + }, + }) +} + +#[cfg(target_os = "macos")] +fn encode_owned_lossless_tiles_to_metal_buffer_fallback_batch( + tiles: &[OwnedMetalLosslessEncodeTile], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, +) -> Result { + let mut outcomes = Vec::with_capacity(tiles.len()); + for tile in tiles { outcomes.push(encode_lossless_tile_to_metal_buffer_with_report( - tile, options, session, staging, + tile.as_tile(), + options, + session, + staging, )?); } Ok(MetalLosslessBufferEncodeBatchOutcome { @@ -1141,20 +1310,25 @@ fn encode_lossless_tiles_to_metal_buffer_batch( } #[cfg(target_os = "macos")] -fn try_encode_resident_lossless_tiles_to_metal_buffer_batch( +fn try_submit_resident_lossless_tiles_to_metal_buffer_batch( tiles: &[MetalLosslessEncodeTile<'_>], options: J2kLosslessEncodeOptions, session: &crate::MetalBackendSession, staging: MetalEncodeInputStaging, config: MetalLosslessEncodeConfig, -) -> Result, crate::Error> { +) -> Result, crate::Error> { + let profile_stages = compute::metal_profile_stages_enabled(); if tiles.is_empty() { - return Ok(Some(MetalLosslessBufferEncodeBatchOutcome { - outcomes: Vec::new(), + return Ok(Some(SubmittedResidentLosslessMetalBufferEncodeBatch { + options, + session: session.clone(), stats: resolve_lossless_encode_config(0, 1, config)?, + encode_started: Instant::now(), + kind: SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty, })); } + let plan_started = profile_stages.then(Instant::now); let mut planned = Vec::with_capacity(tiles.len()); for (index, &tile) in tiles.iter().enumerate() { let Some(item) = plan_resident_lossless_buffer_encode(index, tile, options, staging)? @@ -1168,31 +1342,62 @@ fn try_encode_resident_lossless_tiles_to_metal_buffer_batch( .map(PlannedResidentLosslessBufferEncode::estimated_peak_bytes) .max() .unwrap_or(1); + let uniform_resident_mode = planned.iter().all(|planned| { + planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::HighThroughput + }) || planned + .iter() + .all(|planned| planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::Classic); + if !uniform_resident_mode { + return Ok(None); + } let mut stats = resolve_lossless_encode_config(tiles.len(), estimated_peak_bytes_per_tile, config)?; + if let Some(started) = plan_started { + stats.stage_stats.plan_duration = started.elapsed(); + } let encode_started = Instant::now(); - let outcomes = encode_planned_resident_lossless_tiles( + let kind = submit_planned_resident_lossless_tiles( planned, - options, session, stats.effective_inflight_tiles, &mut stats, )?; - stats.encode_wall_duration = encode_started.elapsed(); - Ok(Some(MetalLosslessBufferEncodeBatchOutcome { - outcomes, + Ok(Some(SubmittedResidentLosslessMetalBufferEncodeBatch { + options, + session: session.clone(), stats, + encode_started, + kind, })) } +#[cfg(target_os = "macos")] +fn try_encode_resident_lossless_tiles_to_metal_buffer_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, +) -> Result, crate::Error> { + let Some(submitted) = try_submit_resident_lossless_tiles_to_metal_buffer_batch( + tiles, options, session, staging, config, + )? + else { + return Ok(None); + }; + wait_submitted_resident_lossless_buffer_encode_batch(submitted).map(Some) +} + #[cfg(any(test, target_os = "macos"))] -const GPU_ENCODE_DEFAULT_INFLIGHT_TILES: usize = 64; +const GPU_ENCODE_DEFAULT_INFLIGHT_TILES: usize = 512; #[cfg(any(test, target_os = "macos"))] const GPU_ENCODE_FALLBACK_HW_MEM_BYTES: usize = 8 * 1024 * 1024 * 1024; #[cfg(any(test, target_os = "macos"))] const GPU_ENCODE_MAX_DEFAULT_MEMORY_BUDGET_BYTES: usize = 10 * 1024 * 1024 * 1024; #[cfg(any(test, target_os = "macos"))] const GPU_ENCODE_MEMORY_BUDGET_PERCENT: usize = 40; +#[cfg(any(test, target_os = "macos"))] +const RESIDENT_HT_DEFAULT_CHUNK_CODE_BLOCKS: usize = 131_072; #[cfg(any(test, target_os = "macos"))] fn default_gpu_encode_memory_budget_bytes_for_hw_mem(hw_memsize: usize) -> usize { @@ -1273,6 +1478,7 @@ fn resolve_lossless_encode_config( estimated_peak_bytes_per_tile, max_observed_inflight_tiles: 0, encode_wall_duration: Duration::ZERO, + stage_stats: MetalLosslessEncodeStageStats::default(), }) } @@ -1295,6 +1501,60 @@ fn checked_mul_bytes(lhs: usize, rhs: usize) -> usize { lhs.saturating_mul(rhs) } +#[cfg(any(test, target_os = "macos"))] +fn resident_lossless_code_block_chunk_cap(code_block_counts: &[usize]) -> usize { + code_block_counts + .iter() + .copied() + .max() + .unwrap_or(1) + .max(RESIDENT_HT_DEFAULT_CHUNK_CODE_BLOCKS) +} + +#[cfg(any(test, target_os = "macos"))] +fn resident_lossless_chunk_ranges_from_code_blocks( + code_block_counts: &[usize], + max_tiles: usize, + max_code_blocks: usize, +) -> Vec> { + if code_block_counts.is_empty() { + return Vec::new(); + } + let max_tiles = max_tiles.max(1); + let max_code_blocks = max_code_blocks.max(1); + let mut ranges = Vec::new(); + let mut start = 0usize; + while start < code_block_counts.len() { + let mut end = start; + let mut chunk_code_blocks = 0usize; + while end < code_block_counts.len() && end - start < max_tiles { + let next_code_blocks = code_block_counts[end].max(1); + let would_exceed_code_blocks = + end > start && chunk_code_blocks.saturating_add(next_code_blocks) > max_code_blocks; + if would_exceed_code_blocks { + break; + } + chunk_code_blocks = chunk_code_blocks.saturating_add(next_code_blocks); + end += 1; + } + if end == start { + end += 1; + } + ranges.push(start..end); + start = end; + } + ranges +} + +#[cfg(test)] +fn resident_lossless_chunk_ranges_for_test( + code_block_counts: &[usize], + max_tiles: usize, + max_code_blocks: usize, +) -> Vec> { + resident_lossless_chunk_ranges_from_code_blocks(code_block_counts, max_tiles, max_code_blocks) +} + #[cfg(target_os = "macos")] #[derive(Clone, Copy)] struct LosslessSubbandPlan { @@ -1360,15 +1620,26 @@ impl PlannedResidentLosslessBufferEncode { } #[cfg(target_os = "macos")] -enum ResidentLosslessTier1 { - Classic(compute::J2kResidentLosslessTier1CodeBlocks), - HighThroughput(compute::J2kResidentLosslessHtCodeBlocks), +struct SubmittedResidentLosslessMetalBufferEncodeBatch { + options: J2kLosslessEncodeOptions, + session: crate::MetalBackendSession, + stats: MetalLosslessEncodeBatchStats, + encode_started: Instant, + kind: SubmittedResidentLosslessMetalBufferEncodeBatchKind, } #[cfg(target_os = "macos")] -struct SubmittedResidentLosslessBufferEncode { - metadata: ResidentLosslessBufferEncodeMetadata, - pending_codestream: compute::J2kPendingResidentLosslessCodestream, +enum SubmittedResidentLosslessMetalBufferEncodeBatchKind { + Empty, + Chunks(Vec), +} + +#[cfg(target_os = "macos")] +struct SubmittedResidentLosslessMetalBufferEncodeChunk { + metadatas: Vec, + prepare_durations: Vec, + pending: compute::J2kPendingResidentLosslessCodestreamBatch, + batch_started: Instant, } #[cfg(target_os = "macos")] @@ -1413,6 +1684,15 @@ fn lossless_device_encode_levels(width: u32, height: u32, options: J2kLosslessEn }) } +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +struct LosslessDwtLevelPlan { + low_width: u32, + low_height: u32, + high_width: u32, + high_height: u32, +} + #[cfg(target_os = "macos")] #[derive(Clone, Copy)] struct LosslessSubbandInput { @@ -1486,6 +1766,32 @@ fn push_lossless_subband_plan( Ok(()) } +#[cfg(target_os = "macos")] +fn lossless_dwt_level_plans( + width: u32, + height: u32, + num_decomposition_levels: u8, +) -> Vec { + let mut levels = Vec::with_capacity(usize::from(num_decomposition_levels)); + let mut current_width = width; + let mut current_height = height; + for _ in 0..num_decomposition_levels { + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let high_width = current_width / 2; + let high_height = current_height / 2; + levels.push(LosslessDwtLevelPlan { + low_width, + low_height, + high_width, + high_height, + }); + current_width = low_width; + current_height = low_height; + } + levels +} + #[cfg(target_os = "macos")] fn lossless_device_encode_plan( width: u32, @@ -1501,9 +1807,6 @@ fn lossless_device_encode_plan( return Ok(None); } let num_decomposition_levels = lossless_device_encode_levels(width, height, options); - if num_decomposition_levels > 1 { - return Ok(None); - } let progression_order = match options.progression { J2kProgressionOrder::Lrcp => EncodeProgressionOrder::Lrcp, J2kProgressionOrder::Rpcl => EncodeProgressionOrder::Rpcl, @@ -1515,6 +1818,7 @@ fn lossless_device_encode_plan( let mut component_resolutions = Vec::>::new(); for component in 0..components { let mut component_packets = Vec::new(); + let dwt_levels = lossless_dwt_level_plans(width, height, num_decomposition_levels); let mut base_packet = LosslessResolutionPlan { subbands: Vec::new(), }; @@ -1535,10 +1839,9 @@ fn lossless_device_encode_plan( )?; component_packets.push(base_packet); } else { - let low_width = width.div_ceil(2); - let low_height = height.div_ceil(2); - let high_width = width / 2; - let high_height = height / 2; + let final_ll = dwt_levels + .last() + .expect("nonzero DWT level count has a final LL level"); push_lossless_subband_plan( &mut base_packet, &mut code_blocks, @@ -1547,60 +1850,62 @@ fn lossless_device_encode_plan( component: u32::from(component), subband_x: 0, subband_y: 0, - width: low_width, - height: low_height, + width: final_ll.low_width, + height: final_ll.low_height, sub_band_type: J2kSubBandType::LowLow, total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_sub(1), }, )?; component_packets.push(base_packet); - let mut detail_packet = LosslessResolutionPlan { - subbands: Vec::new(), - }; - push_lossless_subband_plan( - &mut detail_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: low_width, - subband_y: 0, - width: high_width, - height: low_height, - sub_band_type: J2kSubBandType::HighLow, - total_bitplanes: guard_bits.saturating_add(bit_depth), - }, - )?; - push_lossless_subband_plan( - &mut detail_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: 0, - subband_y: low_height, - width: low_width, - height: high_height, - sub_band_type: J2kSubBandType::LowHigh, - total_bitplanes: guard_bits.saturating_add(bit_depth), - }, - )?; - push_lossless_subband_plan( - &mut detail_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: low_width, - subband_y: low_height, - width: high_width, - height: high_height, - sub_band_type: J2kSubBandType::HighHigh, - total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_add(1), - }, - )?; - component_packets.push(detail_packet); + for level in dwt_levels.iter().rev().copied() { + let mut detail_packet = LosslessResolutionPlan { + subbands: Vec::new(), + }; + push_lossless_subband_plan( + &mut detail_packet, + &mut code_blocks, + &mut coefficient_offset, + LosslessSubbandInput { + component: u32::from(component), + subband_x: level.low_width, + subband_y: 0, + width: level.high_width, + height: level.low_height, + sub_band_type: J2kSubBandType::HighLow, + total_bitplanes: guard_bits.saturating_add(bit_depth), + }, + )?; + push_lossless_subband_plan( + &mut detail_packet, + &mut code_blocks, + &mut coefficient_offset, + LosslessSubbandInput { + component: u32::from(component), + subband_x: 0, + subband_y: level.low_height, + width: level.low_width, + height: level.high_height, + sub_band_type: J2kSubBandType::LowHigh, + total_bitplanes: guard_bits.saturating_add(bit_depth), + }, + )?; + push_lossless_subband_plan( + &mut detail_packet, + &mut code_blocks, + &mut coefficient_offset, + LosslessSubbandInput { + component: u32::from(component), + subband_x: level.low_width, + subband_y: level.low_height, + width: level.high_width, + height: level.high_height, + sub_band_type: J2kSubBandType::HighHigh, + total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_add(1), + }, + )?; + component_packets.push(detail_packet); + } } component_resolutions.push(component_packets); } @@ -1958,48 +2263,6 @@ fn estimated_tier1_output_bytes(plan: &LosslessDeviceEncodePlan) -> usize { .max(1) } -#[cfg(target_os = "macos")] -fn encode_prepared_resident_lossless_tier1( - prepared: PreparedResidentLosslessBufferEncode, - session: &crate::MetalBackendSession, -) -> Result<(ResidentLosslessBufferEncodeMetadata, ResidentLosslessTier1), crate::Error> { - let tier1 = match prepared.metadata.plan.block_coding_mode { - J2kBlockCodingMode::Classic => ResidentLosslessTier1::Classic( - compute::encode_classic_tier1_prepared_device_code_blocks_resident( - session, - prepared.prepared, - )?, - ), - J2kBlockCodingMode::HighThroughput => ResidentLosslessTier1::HighThroughput( - compute::encode_ht_prepared_device_code_blocks_resident(session, prepared.prepared)?, - ), - }; - - Ok((prepared.metadata, tier1)) -} - -#[cfg(target_os = "macos")] -fn resident_packetization_job_for_metadata( - metadata: &ResidentLosslessBufferEncodeMetadata, -) -> Result, crate::Error> { - Ok(compute::J2kResidentPacketizationEncodeJob { - resolution_count: u32::try_from(metadata.plan.resolutions.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode resolution count exceeds u32".to_string(), - } - })?, - num_layers: 1, - num_components: metadata.plan.components, - code_block_count: u32::try_from(metadata.plan.code_blocks.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block count exceeds u32".to_string(), - } - })?, - packet_descriptors: &metadata.packet_descriptors, - resolutions: &metadata.packetization_resolutions, - }) -} - #[cfg(target_os = "macos")] fn resident_codestream_assembly_job_for_metadata( metadata: &ResidentLosslessBufferEncodeMetadata, @@ -2025,51 +2288,58 @@ fn resident_codestream_assembly_job_for_metadata( } #[cfg(target_os = "macos")] -fn submit_resident_lossless_buffer_encode( - metadata: ResidentLosslessBufferEncodeMetadata, - tier1: &ResidentLosslessTier1, - session: &crate::MetalBackendSession, -) -> Result { - let packetization_job = resident_packetization_job_for_metadata(&metadata)?; - let assembly_job = resident_codestream_assembly_job_for_metadata(&metadata); - let pending_codestream = match tier1 { - ResidentLosslessTier1::Classic(tier1) => { - compute::submit_lossless_codestream_buffer_from_resident_classic_tier1( - session, - tier1, - packetization_job, - assembly_job, - )? - } - ResidentLosslessTier1::HighThroughput(tier1) => { - compute::submit_lossless_codestream_buffer_from_resident_ht_tier1( - session, - tier1, - packetization_job, - assembly_job, - )? +fn wait_submitted_resident_lossless_buffer_encode_batch( + mut submitted: SubmittedResidentLosslessMetalBufferEncodeBatch, +) -> Result { + let mut outcomes = Vec::new(); + match submitted.kind { + SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty => {} + SubmittedResidentLosslessMetalBufferEncodeBatchKind::Chunks(chunks) => { + outcomes.reserve(chunks.iter().map(|chunk| chunk.metadatas.len()).sum()); + for chunk in chunks { + let wait_started = compute::metal_profile_stages_enabled().then(Instant::now); + let batch = compute::wait_resident_lossless_codestream_batch(chunk.pending)?; + if let Some(started) = wait_started { + submitted.stats.stage_stats.codestream_wait_duration = submitted + .stats + .stage_stats + .codestream_wait_duration + .saturating_add(started.elapsed()); + submitted + .stats + .stage_stats + .add_assign(MetalLosslessEncodeStageStats::from(batch.stage_stats)); + } + let codestreams = batch.codestreams; + let batch_duration = + duration_share(chunk.batch_started.elapsed(), codestreams.len()); + for ((metadata, prepare_duration), codestream) in chunk + .metadatas + .into_iter() + .zip(chunk.prepare_durations) + .zip(codestreams) + { + let finished = finished_resident_lossless_buffer_encode( + metadata, + codestream, + prepare_duration.saturating_add(batch_duration), + ); + outcomes.push(validate_finished_resident_lossless_buffer_encode( + finished, + submitted.options, + &submitted.session, + )?); + } + } } - }; - Ok(SubmittedResidentLosslessBufferEncode { - metadata, - pending_codestream, + } + submitted.stats.encode_wall_duration = submitted.encode_started.elapsed(); + Ok(MetalLosslessBufferEncodeBatchOutcome { + outcomes, + stats: submitted.stats, }) } -#[cfg(target_os = "macos")] -fn wait_submitted_resident_lossless_buffer_encode( - submitted: SubmittedResidentLosslessBufferEncode, -) -> Result { - let encode_started = Instant::now(); - let codestream = compute::wait_resident_lossless_codestream(submitted.pending_codestream)?; - let encode_duration = encode_started.elapsed(); - Ok(finished_resident_lossless_buffer_encode( - submitted.metadata, - codestream, - encode_duration, - )) -} - #[cfg(target_os = "macos")] fn finished_resident_lossless_buffer_encode( metadata: ResidentLosslessBufferEncodeMetadata, @@ -2223,22 +2493,20 @@ where } #[cfg(target_os = "macos")] -fn encode_planned_resident_lossless_tiles( +fn submit_planned_resident_lossless_tiles( planned: Vec, - options: J2kLosslessEncodeOptions, session: &crate::MetalBackendSession, inflight_tiles: usize, stats: &mut MetalLosslessEncodeBatchStats, -) -> Result, crate::Error> { +) -> Result { if planned.is_empty() { - return Ok(Vec::new()); + return Ok(SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty); } if planned.iter().all(|planned| { planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::HighThroughput }) { - return encode_planned_resident_ht_lossless_tiles_batch( + return submit_planned_resident_ht_lossless_tiles_batch( planned, - options, session, inflight_tiles, stats, @@ -2248,27 +2516,14 @@ fn encode_planned_resident_lossless_tiles( .iter() .all(|planned| planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::Classic) { - return encode_planned_resident_classic_lossless_tiles_batch( + return submit_planned_resident_classic_lossless_tiles_batch( planned, - options, session, inflight_tiles, stats, ); } - - let encoded = collect_inflight_limited_ordered(planned, inflight_tiles, |_, planned| { - let index = planned.index; - encode_planned_resident_lossless_tile(planned, options, session).map_err(|err| { - crate::Error::MetalKernel { - message: format!("J2K Metal resident encode failed at tile {index}: {err}"), - } - }) - })?; - stats.max_observed_inflight_tiles = stats - .max_observed_inflight_tiles - .max(encoded.max_observed_inflight_items); - Ok(encoded.items) + Ok(SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty) } #[cfg(target_os = "macos")] @@ -2348,86 +2603,125 @@ fn prepare_planned_resident_ht_lossless_tiles_batch( } #[cfg(target_os = "macos")] -fn encode_planned_resident_ht_lossless_tiles_batch( - planned: Vec, - options: J2kLosslessEncodeOptions, +fn submit_planned_resident_ht_lossless_tiles_batch( + mut planned: Vec, session: &crate::MetalBackendSession, inflight_tiles: usize, stats: &mut MetalLosslessEncodeBatchStats, -) -> Result, crate::Error> { +) -> Result { let planned_len = planned.len(); - let prepared = - prepare_planned_resident_ht_lossless_tiles_batch(planned, session).map_err(|err| { - crate::Error::MetalKernel { - message: format!("J2K Metal resident HT batch encode failed: {err}"), - } - })?; + let profile_stages = compute::metal_profile_stages_enabled(); + let code_block_counts = planned + .iter() + .map(|planned| planned.metadata.plan.code_blocks.len()) + .collect::>(); + let chunk_ranges = resident_lossless_chunk_ranges_from_code_blocks( + &code_block_counts, + inflight_tiles, + resident_lossless_code_block_chunk_cap(&code_block_counts), + ); + if profile_stages { + stats.stage_stats.chunk_count = stats + .stage_stats + .chunk_count + .saturating_add(chunk_ranges.len()); + stats.stage_stats.tile_count = stats.stage_stats.tile_count.saturating_add(planned_len); + } stats.max_observed_inflight_tiles = stats.max_observed_inflight_tiles.max( - planned_len - .min(inflight_tiles) - .max(usize::from(planned_len > 0)), + chunk_ranges + .iter() + .map(std::ops::Range::len) + .max() + .unwrap_or(0), ); - let mut metadatas = Vec::with_capacity(prepared.len()); - let mut prepare_durations = Vec::with_capacity(prepared.len()); - let mut batch_items = Vec::with_capacity(prepared.len()); - for item in prepared { - let PreparedResidentLosslessBatchItem { - prepared, - prepare_duration, - } = item; - let metadata = prepared.metadata; - let codestream = resident_codestream_assembly_job_for_metadata(&metadata); - batch_items.push(compute::J2kResidentHtBatchEncodeItem { - prepared: prepared.prepared, - resolution_count: u32::try_from(metadata.plan.resolutions.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode resolution count exceeds u32".to_string(), - } - })?, - num_layers: 1, - num_components: metadata.plan.components, - code_block_count: u32::try_from(metadata.plan.code_blocks.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block count exceeds u32".to_string(), - } - })?, - packet_descriptors: metadata.packet_descriptors.clone(), - resolutions: metadata.packetization_resolutions.clone(), - codestream, + let mut chunks = Vec::with_capacity(chunk_ranges.len()); + for range in chunk_ranges { + let take = range.len(); + let chunk_planned = planned.drain(..take).collect::>(); + let prepare_submit_started = profile_stages.then(Instant::now); + let prepared = prepare_planned_resident_ht_lossless_tiles_batch(chunk_planned, session) + .map_err(|err| crate::Error::MetalKernel { + message: format!("J2K Metal resident HT batch encode failed: {err}"), + })?; + + let mut metadatas = Vec::with_capacity(prepared.len()); + let mut prepare_durations = Vec::with_capacity(prepared.len()); + let mut batch_items = Vec::with_capacity(prepared.len()); + for item in prepared { + let PreparedResidentLosslessBatchItem { + prepared, + prepare_duration, + } = item; + let metadata = prepared.metadata; + let codestream = resident_codestream_assembly_job_for_metadata(&metadata); + batch_items.push(compute::J2kResidentHtBatchEncodeItem { + prepared: prepared.prepared, + resolution_count: u32::try_from(metadata.plan.resolutions.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode resolution count exceeds u32" + .to_string(), + } + })?, + num_layers: 1, + num_components: metadata.plan.components, + code_block_count: u32::try_from(metadata.plan.code_blocks.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block count exceeds u32" + .to_string(), + } + })?, + packet_descriptors: metadata.packet_descriptors.clone(), + resolutions: metadata.packetization_resolutions.clone(), + codestream, + }); + prepare_durations.push(prepare_duration); + metadatas.push(metadata); + } + + let batch_started = Instant::now(); + let pending = compute::submit_lossless_codestream_buffers_from_prepared_ht_batch( + session, + batch_items, + )?; + if let Some(started) = prepare_submit_started { + stats.stage_stats.prepare_submit_duration = stats + .stage_stats + .prepare_submit_duration + .saturating_add(started.elapsed()); + } + chunks.push(SubmittedResidentLosslessMetalBufferEncodeChunk { + metadatas, + prepare_durations, + pending, + batch_started, }); - prepare_durations.push(prepare_duration); - metadatas.push(metadata); } - let batch_started = Instant::now(); - let pending = - compute::submit_lossless_codestream_buffers_from_prepared_ht_batch(session, batch_items)?; - let codestreams = compute::wait_resident_lossless_codestream_batch(pending)?; - let batch_duration = duration_share(batch_started.elapsed(), codestreams.len()); - metadatas - .into_iter() - .zip(prepare_durations) - .zip(codestreams) - .map(|((metadata, prepare_duration), codestream)| { - let finished = finished_resident_lossless_buffer_encode( - metadata, - codestream, - prepare_duration.saturating_add(batch_duration), - ); - validate_finished_resident_lossless_buffer_encode(finished, options, session) - }) - .collect() + if !planned.is_empty() { + return Err(crate::Error::MetalKernel { + message: "J2K Metal resident HT batch chunking left unsubmitted tiles".to_string(), + }); + } + + if chunks.is_empty() && planned_len > 0 { + return Err(crate::Error::MetalKernel { + message: "J2K Metal resident HT batch chunking produced no chunks".to_string(), + }); + } + + Ok(SubmittedResidentLosslessMetalBufferEncodeBatchKind::Chunks( + chunks, + )) } #[cfg(target_os = "macos")] -fn encode_planned_resident_classic_lossless_tiles_batch( +fn submit_planned_resident_classic_lossless_tiles_batch( planned: Vec, - options: J2kLosslessEncodeOptions, session: &crate::MetalBackendSession, inflight_tiles: usize, stats: &mut MetalLosslessEncodeBatchStats, -) -> Result, crate::Error> { +) -> Result { let prepared = collect_inflight_limited_ordered(planned, inflight_tiles, |_, planned| { let index = planned.index; let started = Instant::now(); @@ -2477,7 +2771,7 @@ fn encode_planned_resident_classic_lossless_tiles_batch( } let batch_limit = inflight_tiles.max(1); - let mut outcomes = Vec::with_capacity(metadatas.len()); + let mut chunks = Vec::new(); while !batch_items.is_empty() { let take = batch_items.len().min(batch_limit); let chunk_items = batch_items.drain(..take).collect(); @@ -2488,24 +2782,16 @@ fn encode_planned_resident_classic_lossless_tiles_batch( session, chunk_items, )?; - let codestreams = compute::wait_resident_lossless_codestream_batch(pending)?; - let batch_duration = duration_share(batch_started.elapsed(), codestreams.len()); - for ((metadata, prepare_duration), codestream) in chunk_metadatas - .into_iter() - .zip(chunk_prepare_durations) - .zip(codestreams) - { - let finished = finished_resident_lossless_buffer_encode( - metadata, - codestream, - prepare_duration.saturating_add(batch_duration), - ); - outcomes.push(validate_finished_resident_lossless_buffer_encode( - finished, options, session, - )?); - } + chunks.push(SubmittedResidentLosslessMetalBufferEncodeChunk { + metadatas: chunk_metadatas, + prepare_durations: chunk_prepare_durations, + pending, + batch_started, + }); } - Ok(outcomes) + Ok(SubmittedResidentLosslessMetalBufferEncodeBatchKind::Chunks( + chunks, + )) } #[cfg(target_os = "macos")] @@ -2584,21 +2870,6 @@ impl Drop for ActiveTileGuard<'_> { } } -#[cfg(target_os = "macos")] -fn encode_planned_resident_lossless_tile( - planned: PlannedResidentLosslessBufferEncode, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let encode_started = Instant::now(); - let prepared = prepare_planned_resident_lossless_tile(planned, session)?; - let (metadata, tier1) = encode_prepared_resident_lossless_tier1(prepared, session)?; - let submitted = submit_resident_lossless_buffer_encode(metadata, &tier1, session)?; - let mut finished = wait_submitted_resident_lossless_buffer_encode(submitted)?; - finished.encode_duration = encode_started.elapsed(); - validate_finished_resident_lossless_buffer_encode(finished, options, session) -} - #[cfg(all(test, target_os = "macos"))] thread_local! { static TEST_RESIDENT_ENCODE_FAILURE_INDEX: Cell> = const { Cell::new(None) }; @@ -3205,6 +3476,17 @@ pub fn encode_lossless_from_metal_buffers_to_metal_batch( Err(crate::Error::MetalUnavailable) } +#[cfg(not(target_os = "macos"))] +pub fn submit_lossless_from_metal_buffers_to_metal_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + config: MetalLosslessEncodeConfig, +) -> Result { + let _ = (tiles, options, session, config); + Err(crate::Error::MetalUnavailable) +} + #[cfg(not(target_os = "macos"))] pub fn encode_lossless_from_padded_metal_buffers( tiles: &[MetalLosslessEncodeTile<'_>], @@ -3266,6 +3548,17 @@ pub fn encode_lossless_from_padded_metal_buffers_to_metal_batch( Err(crate::Error::MetalUnavailable) } +#[cfg(not(target_os = "macos"))] +pub fn submit_lossless_from_padded_metal_buffers_to_metal_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + config: MetalLosslessEncodeConfig, +) -> Result { + let _ = (tiles, options, session, config); + Err(crate::Error::MetalUnavailable) +} + #[cfg(not(target_os = "macos"))] pub fn encode_lossless_from_padded_metal_buffers_to_metal_with_report( tiles: &[MetalLosslessEncodeTile<'_>], @@ -3627,6 +3920,69 @@ mod tests { assert_submit_batch_fn(crate::submit_lossless_from_padded_metal_buffers); } + #[test] + fn submitted_lossless_metal_buffer_encode_public_api_is_available() { + fn assert_buffer_batch_submission< + S: DeviceSubmission< + Output = super::MetalLosslessBufferEncodeBatchOutcome, + Error = crate::Error, + >, + >() { + } + fn assert_submit_buffer_batch_fn( + _submit: for<'slice, 'tile, 'options, 'session> fn( + &'slice [super::MetalLosslessEncodeTile<'tile>], + &'options J2kLosslessEncodeOptions, + &'session crate::MetalBackendSession, + super::MetalLosslessEncodeConfig, + ) -> Result< + crate::SubmittedJ2kLosslessMetalBufferEncodeBatch, + crate::Error, + >, + ) { + } + + assert_buffer_batch_submission::(); + assert_submit_buffer_batch_fn(crate::submit_lossless_from_metal_buffers_to_metal_batch); + assert_submit_buffer_batch_fn( + crate::submit_lossless_from_padded_metal_buffers_to_metal_batch, + ); + } + + #[test] + fn resident_lossless_stage_stats_default_to_zero() { + let stats = super::MetalLosslessEncodeBatchStats::default(); + + assert_eq!( + stats.stage_stats, + super::MetalLosslessEncodeStageStats::default() + ); + assert!(!stats.stage_stats.has_timings()); + } + + #[test] + fn resident_lossless_chunk_ranges_respect_inflight_and_code_block_caps() { + assert_eq!( + super::resident_lossless_chunk_ranges_for_test(&[32, 32, 32, 32, 32], 3, 96), + vec![0..3, 3..5] + ); + assert_eq!( + super::resident_lossless_chunk_ranges_for_test(&[80, 80, 10], 8, 96), + vec![0..1, 1..3] + ); + } + + #[test] + fn resident_lossless_default_code_block_cap_allows_large_wsi_chunks() { + let code_blocks = vec![192usize; 600]; + let cap = super::resident_lossless_code_block_chunk_cap(&code_blocks); + + assert_eq!( + super::resident_lossless_chunk_ranges_for_test(&code_blocks, 512, cap), + vec![0..512, 512..600] + ); + } + #[cfg(target_os = "macos")] #[test] fn metal_dispatch_option_treats_unavailable_as_no_dispatch() { @@ -4375,6 +4731,126 @@ mod tests { assert_eq!(decoded.data, pixels); } + #[cfg(target_os = "macos")] + #[test] + fn metal_padded_private_rgb8_ht_rpcl_512_encode_preserves_three_dwt_levels_and_stays_resident() + { + let pixels: Vec = (0..512 * 512 * 3) + .map(|idx| ((idx * 47 + idx / 17) & 0xFF) as u8) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_to_metal_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 512, + height: 512, + pitch_bytes: 512 * 3, + output_width: 512, + output_height: 512, + format: PixelFormat::Rgb8, + }, + &J2kLosslessEncodeOptions { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + progression: J2kProgressionOrder::Rpcl, + ..J2kLosslessEncodeOptions::default() + }, + &session, + ) + .expect("Metal private padded HTJ2K RPCL 512 buffer lossless encode"); + + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let codestream = encoded + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let cod_marker = codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(codestream[cod_marker + 5], 0x02); + assert_eq!(codestream[cod_marker + 9], 3); + assert_eq!(codestream[cod_marker + 12], 0x40); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, pixels); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_rgb8_ht_batch_uses_fused_deinterleave_rct_kernel() { + const WIDTH: usize = 32; + const HEIGHT: usize = 32; + let first: Vec = (0..WIDTH * HEIGHT * 3) + .map(|idx| ((idx * 29 + idx / 7) & 0xFF) as u8) + .collect(); + let second: Vec = (0..WIDTH * HEIGHT * 3) + .map(|idx| 255u8.wrapping_sub(((idx * 13 + idx / 5) & 0xFF) as u8)) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: WIDTH as u32, + height: HEIGHT as u32, + pitch_bytes: WIDTH * 3, + output_width: WIDTH as u32, + output_height: HEIGHT as u32, + format: PixelFormat::Rgb8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: WIDTH as u32, + height: HEIGHT as u32, + pitch_bytes: WIDTH * 3, + output_width: WIDTH as u32, + output_height: HEIGHT as u32, + format: PixelFormat::Rgb8, + }, + ]; + let options = J2kLosslessEncodeOptions { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + ..J2kLosslessEncodeOptions::default() + }; + + compute::reset_lossless_deinterleave_rct_fused_dispatches_for_test(); + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, &options, &session, + ) + .expect("Metal RGB8 HTJ2K batch encode"); + + assert_eq!(encoded.len(), 2); + assert!( + compute::lossless_deinterleave_rct_fused_dispatches_for_test() > 0, + "RGB8 resident lossless encode should fuse deinterleave and RCT" + ); + for (frame, expected) in encoded.iter().zip([first, second]) { + let codestream = frame + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, expected); + } + } + #[cfg(target_os = "macos")] #[test] fn metal_buffer_lossless_batch_encodes_padded_contiguous_inputs() { @@ -4846,9 +5322,9 @@ mod tests { } #[test] - fn gpu_encode_default_inflight_uses_sixty_four_when_memory_allows() { + fn gpu_encode_default_inflight_uses_large_wsi_batch_when_memory_allows() { let stats = super::resolve_lossless_encode_config_for_test( - 100, + 600, 1_000, super::MetalLosslessEncodeConfig { gpu_encode_inflight_tiles: None, @@ -4858,7 +5334,7 @@ mod tests { .expect("resolved config"); assert_eq!(stats.configured_inflight_tiles, None); - assert_eq!(stats.effective_inflight_tiles, 64); + assert_eq!(stats.effective_inflight_tiles, 512); } #[test] diff --git a/crates/signinum-j2k-metal/src/lib.rs b/crates/signinum-j2k-metal/src/lib.rs index 0e8c58ed..bb6487c0 100644 --- a/crates/signinum-j2k-metal/src/lib.rs +++ b/crates/signinum-j2k-metal/src/lib.rs @@ -71,13 +71,16 @@ pub use encode::{ encode_lossless_from_padded_metal_buffers_to_metal_batch, encode_lossless_from_padded_metal_buffers_to_metal_with_report, encode_lossless_from_padded_metal_buffers_with_report, submit_lossless_from_metal_buffer, - submit_lossless_from_metal_buffers, submit_lossless_from_metal_buffers_with_config, - submit_lossless_from_padded_metal_buffer, submit_lossless_from_padded_metal_buffers, + submit_lossless_from_metal_buffers, submit_lossless_from_metal_buffers_to_metal_batch, + submit_lossless_from_metal_buffers_with_config, submit_lossless_from_padded_metal_buffer, + submit_lossless_from_padded_metal_buffers, + submit_lossless_from_padded_metal_buffers_to_metal_batch, submit_lossless_from_padded_metal_buffers_with_config, validate_lossless_roundtrip_on_metal, validate_lossless_roundtrip_on_metal_with_session, MetalEncodeStageAccelerator, MetalEncodedJ2k, MetalLosslessBufferEncodeBatchOutcome, MetalLosslessBufferEncodeOutcome, MetalLosslessEncodeBatchStats, MetalLosslessEncodeConfig, MetalLosslessEncodeOutcome, - MetalLosslessEncodeResidency, MetalLosslessEncodeTile, SubmittedJ2kLosslessMetalEncode, + MetalLosslessEncodeResidency, MetalLosslessEncodeStageStats, MetalLosslessEncodeTile, + SubmittedJ2kLosslessMetalBufferEncodeBatch, SubmittedJ2kLosslessMetalEncode, SubmittedJ2kLosslessMetalEncodeBatch, }; diff --git a/crates/signinum-j2k-metal/src/mct.metal b/crates/signinum-j2k-metal/src/mct.metal index e256dda1..b323677d 100644 --- a/crates/signinum-j2k-metal/src/mct.metal +++ b/crates/signinum-j2k-metal/src/mct.metal @@ -92,3 +92,57 @@ kernel void j2k_forward_rct( status->detail = 0; } } + +kernel void j2k_lossless_deinterleave_rct_rgb8_to_planes( + device const uchar *src [[buffer(0)]], + device float *plane0 [[buffer(1)]], + device float *plane1 [[buffer(2)]], + device float *plane2 [[buffer(3)]], + constant J2kLosslessDeinterleaveParams ¶ms [[buffer(4)]], + device J2kMctStatus *status [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.dst_width || gid.y >= params.dst_height) { + return; + } + + const bool inside_src = gid.x < params.src_width && gid.y < params.src_height; + const uint src_base = gid.y * params.src_stride + gid.x * 3u; + const uint dst_idx = gid.y * params.dst_width + gid.x; + const float r = j2k_lossless_load_sample( + src, + src_base, + 0u, + 3u, + 1u, + params.sample_offset, + inside_src + ); + const float g = j2k_lossless_load_sample( + src, + src_base, + 1u, + 3u, + 1u, + params.sample_offset, + inside_src + ); + const float b = j2k_lossless_load_sample( + src, + src_base, + 2u, + 3u, + 1u, + params.sample_offset, + inside_src + ); + + plane0[dst_idx] = floor((r + 2.0f * g + b) * 0.25f); + plane1[dst_idx] = b - g; + plane2[dst_idx] = r - g; + + if (gid.x == 0u && gid.y == 0u) { + status->code = J2K_MCT_STATUS_OK; + status->detail = 0; + } +} From d47e503c3998dff524b4b8fe566c2c789fb39aa1 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 17 May 2026 10:22:42 -0400 Subject: [PATCH 002/271] bench: add J2K perf guard --- Cargo.lock | 3 + .../benches/tier1_bitplane.rs | 74 +++- .../tests/bench_harness.rs | 20 + crates/signinum-j2k/benches/public_api.rs | 184 ++++++++- crates/signinum-j2k/tests/bench_harness.rs | 22 ++ docs/bench.md | 13 + xtask/Cargo.toml | 3 + xtask/src/main.rs | 116 ++++++ xtask/src/perf_guard.rs | 358 ++++++++++++++++++ 9 files changed, 787 insertions(+), 6 deletions(-) create mode 100644 crates/signinum-j2k-native/tests/bench_harness.rs create mode 100644 crates/signinum-j2k/tests/bench_harness.rs create mode 100644 xtask/src/perf_guard.rs diff --git a/Cargo.lock b/Cargo.lock index 5f2543ff..04a40a8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1326,6 +1326,9 @@ dependencies = [ [[package]] name = "xtask" version = "0.0.0" +dependencies = [ + "serde_json", +] [[package]] name = "zerocopy" diff --git a/crates/signinum-j2k-native/benches/tier1_bitplane.rs b/crates/signinum-j2k-native/benches/tier1_bitplane.rs index 0cfcbda6..47d6f5ba 100644 --- a/crates/signinum-j2k-native/benches/tier1_bitplane.rs +++ b/crates/signinum-j2k-native/benches/tier1_bitplane.rs @@ -1,6 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; use signinum_j2k_native::{ - decode_j2k_code_block_scalar, encode_j2k_code_block_scalar_with_style, J2kCodeBlockDecodeJob, + decode_ht_code_block_scalar, decode_j2k_code_block_scalar, encode_ht_code_block_scalar, + encode_j2k_code_block_scalar_with_style, HtCodeBlockDecodeJob, J2kCodeBlockDecodeJob, J2kCodeBlockStyle, J2kSubBandType, }; @@ -163,5 +164,74 @@ fn bench_tier1_encode(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_tier1_decode, bench_tier1_encode); +fn bench_htj2k_cleanup_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_decode"); + for &seed in &[0x9191_0000, 0x9191_0001] { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, seed); + let encoded = encode_ht_code_block_scalar(&coefficients, width, height, total_bitplanes) + .expect("encode HTJ2K code block"); + let job = HtCodeBlockDecodeJob { + data: &encoded.data, + cleanup_length: encoded.data.len() as u32, + refinement_length: 0, + width, + height, + output_stride: width as usize, + missing_bit_planes: encoded.num_zero_bitplanes, + number_of_coding_passes: encoded.num_coding_passes, + num_bitplanes: total_bitplanes, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }; + let mut output = vec![0.0; width as usize * height as usize]; + + group.bench_with_input(BenchmarkId::new("decode_64x64", seed), &job, |b, job| { + b.iter(|| { + decode_ht_code_block_scalar(*job, &mut output).expect("decode HTJ2K code block"); + black_box(&output); + }); + }); + } + group.finish(); +} + +fn bench_htj2k_cleanup_encode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode"); + for &seed in &[0x9292_0000, 0x9292_0001] { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, seed); + + group.bench_with_input( + BenchmarkId::new("encode_64x64", seed), + &coefficients, + |b, coefficients| { + b.iter(|| { + let encoded = encode_ht_code_block_scalar( + black_box(coefficients), + width, + height, + total_bitplanes, + ) + .expect("encode HTJ2K code block"); + black_box(encoded); + }); + }, + ); + } + group.finish(); +} + +criterion_group!( + benches, + bench_tier1_decode, + bench_tier1_encode, + bench_htj2k_cleanup_decode, + bench_htj2k_cleanup_encode +); criterion_main!(benches); diff --git a/crates/signinum-j2k-native/tests/bench_harness.rs b/crates/signinum-j2k-native/tests/bench_harness.rs new file mode 100644 index 00000000..7a58e1c6 --- /dev/null +++ b/crates/signinum-j2k-native/tests/bench_harness.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn native_bench_exposes_classic_and_htj2k_code_block_surface() { + let bench = include_str!("../benches/tier1_bitplane.rs"); + + for expected in [ + "tier1_bitplane_decode", + "tier1_bitplane_encode", + "htj2k_cleanup_decode", + "htj2k_cleanup_encode", + "decode_64x64", + "encode_64x64", + ] { + assert!( + bench.contains(expected), + "native benchmark is missing `{expected}`" + ); + } +} diff --git a/crates/signinum-j2k/benches/public_api.rs b/crates/signinum-j2k/benches/public_api.rs index d926c7c9..38d7f4e7 100644 --- a/crates/signinum-j2k/benches/public_api.rs +++ b/crates/signinum-j2k/benches/public_api.rs @@ -2,13 +2,17 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use signinum_j2k::{ - encode_j2k_lossless, EncodeBackendPreference, J2kDecoder, J2kEncodeValidation, - J2kLosslessEncodeOptions, J2kLosslessSamples, J2kScratchPool, PixelFormat, Rect, + encode_j2k_lossless, DecoderContext, Downscale, EncodeBackendPreference, ImageDecodeRows, + J2kBlockCodingMode, J2kCodec, J2kContext, J2kDecoder, J2kEncodeValidation, + J2kLosslessEncodeOptions, J2kLosslessSamples, J2kScratchPool, PixelFormat, Rect, RowSink, + TileBatchDecode, }; use signinum_test_support::{patterned_gray8, patterned_rgb8}; const TILE_SIDE: u32 = 128; const ROI_SIDE: u32 = 64; +const HT_TILE_SIDE: u32 = 512; +const BATCH_SIZE: usize = 16; fn bench_encode_options() -> J2kLosslessEncodeOptions { J2kLosslessEncodeOptions { @@ -20,9 +24,31 @@ fn bench_encode_options() -> J2kLosslessEncodeOptions { fn encode_gray8_codestream(width: u32, height: u32) -> Vec { let pixels = patterned_gray8(width, height); + encode_gray8_codestream_from_pixels(width, height, &pixels, bench_encode_options()) +} + +fn encode_ht_gray8_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_gray8(width, height); + encode_gray8_codestream_from_pixels( + width, + height, + &pixels, + J2kLosslessEncodeOptions { + block_coding_mode: J2kBlockCodingMode::HighThroughput, + ..bench_encode_options() + }, + ) +} + +fn encode_gray8_codestream_from_pixels( + width: u32, + height: u32, + pixels: &[u8], + options: J2kLosslessEncodeOptions, +) -> Vec { let samples = - J2kLosslessSamples::new(&pixels, width, height, 1, 8, false).expect("valid gray8 samples"); - encode_j2k_lossless(samples, &bench_encode_options()) + J2kLosslessSamples::new(pixels, width, height, 1, 8, false).expect("valid gray8 samples"); + encode_j2k_lossless(samples, &options) .expect("encode gray8 codestream") .codestream } @@ -92,6 +118,7 @@ fn bench_inspect(c: &mut Criterion) { fn bench_decode(c: &mut Criterion) { let codestream = encode_rgb8_codestream(TILE_SIDE, TILE_SIDE); + let ht_codestream = encode_ht_gray8_codestream(HT_TILE_SIDE, HT_TILE_SIDE); let mut group = c.benchmark_group("j2k_public_decode"); let full_stride = TILE_SIDE as usize * 3; @@ -127,6 +154,127 @@ fn bench_decode(c: &mut Criterion) { }); }); + let ht_stride = HT_TILE_SIDE as usize; + let mut ht_out = vec![0u8; ht_stride * HT_TILE_SIDE as usize]; + group.bench_function("htj2k_gray8_full_512x512", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(black_box(ht_codestream.as_slice())).expect("htj2k decoder"); + decoder + .decode_into(&mut ht_out, ht_stride, PixelFormat::Gray8) + .expect("decode full htj2k gray8"); + black_box(&ht_out); + }); + }); + + group.finish(); +} + +fn bench_region_scaled(c: &mut Criterion) { + let codestream = encode_rgb8_codestream(TILE_SIDE, TILE_SIDE); + let roi = Rect { + x: 32, + y: 32, + w: ROI_SIDE, + h: ROI_SIDE, + }; + let out_side = ROI_SIDE.div_ceil(Downscale::Quarter.denominator()); + let stride = out_side as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut out = vec![0u8; stride * out_side as usize]; + let mut pool = J2kScratchPool::new(); + + let mut group = c.benchmark_group("j2k_public_decode_region_scaled"); + group.bench_function("rgb8_region_scaled_64x64_q4", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(black_box(codestream.as_slice())).expect("rgb8 decoder"); + decoder + .decode_region_scaled_into( + &mut pool, + &mut out, + stride, + PixelFormat::Rgb8, + roi, + Downscale::Quarter, + ) + .expect("decode rgb8 region scaled"); + black_box(&out); + }); + }); + group.finish(); +} + +fn bench_rows(c: &mut Criterion) { + let codestream = encode_gray8_codestream(TILE_SIDE, TILE_SIDE); + let mut group = c.benchmark_group("j2k_public_decode_rows"); + group.bench_function("gray8_rows_128x128", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(black_box(codestream.as_slice())).expect("gray8 decoder"); + let mut sink = VecRowSink::new(TILE_SIDE, TILE_SIDE); + decoder.decode_rows(&mut sink).expect("decode gray8 rows"); + black_box(sink.rows); + }); + }); + group.finish(); +} + +fn bench_tile_batch(c: &mut Criterion) { + let repeated = encode_gray8_codestream(TILE_SIDE, TILE_SIDE); + let mut distinct = Vec::with_capacity(BATCH_SIZE); + for idx in 0..BATCH_SIZE { + let mut pixels = patterned_gray8(TILE_SIDE, TILE_SIDE); + pixels[0] = pixels[0].wrapping_add(idx as u8); + distinct.push(encode_gray8_codestream_from_pixels( + TILE_SIDE, + TILE_SIDE, + &pixels, + bench_encode_options(), + )); + } + + let stride = TILE_SIDE as usize; + let mut out = vec![0u8; stride * TILE_SIDE as usize]; + let mut group = c.benchmark_group("j2k_public_tile_batch"); + + group.bench_function("gray8_repeated_batch_16", |b| { + b.iter(|| { + let mut ctx = DecoderContext::::default(); + let mut pool = J2kScratchPool::new(); + for _ in 0..BATCH_SIZE { + ::decode_tile( + &mut ctx, + &mut pool, + black_box(repeated.as_slice()), + &mut out, + stride, + PixelFormat::Gray8, + ) + .expect("decode repeated gray8 tile"); + } + black_box(&out); + }); + }); + + group.bench_function("gray8_distinct_batch_16", |b| { + b.iter(|| { + let mut ctx = DecoderContext::::default(); + let mut pool = J2kScratchPool::new(); + for codestream in &distinct { + ::decode_tile( + &mut ctx, + &mut pool, + black_box(codestream.as_slice()), + &mut out, + stride, + PixelFormat::Gray8, + ) + .expect("decode distinct gray8 tile"); + } + black_box(&out); + }); + }); + group.finish(); } @@ -154,6 +302,34 @@ criterion_group!( bench_lossless_encode, bench_inspect, bench_decode, + bench_region_scaled, + bench_rows, + bench_tile_batch, bench_decode_gray_setup ); criterion_main!(benches); + +struct VecRowSink { + rows: Vec, + width: usize, +} + +impl VecRowSink { + fn new(width: u32, height: u32) -> Self { + Self { + rows: vec![0; width as usize * height as usize], + width: width as usize, + } + } +} + +impl RowSink for VecRowSink { + type Error = std::convert::Infallible; + + fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), Self::Error> { + let start = y as usize * self.width; + let end = start + row.len(); + self.rows[start..end].copy_from_slice(row); + Ok(()) + } +} diff --git a/crates/signinum-j2k/tests/bench_harness.rs b/crates/signinum-j2k/tests/bench_harness.rs new file mode 100644 index 00000000..37a05a11 --- /dev/null +++ b/crates/signinum-j2k/tests/bench_harness.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn public_api_bench_exposes_cpu_decode_regression_surface() { + let bench = include_str!("../benches/public_api.rs"); + + for expected in [ + "j2k_public_decode_region_scaled", + "j2k_public_decode_rows", + "j2k_public_tile_batch", + "rgb8_region_scaled_64x64_q4", + "gray8_rows_128x128", + "gray8_repeated_batch_16", + "gray8_distinct_batch_16", + "htj2k_gray8_full_512x512", + ] { + assert!( + bench.contains(expected), + "public API benchmark is missing `{expected}`" + ); + } +} diff --git a/docs/bench.md b/docs/bench.md index 98e109b1..e4c60e39 100644 --- a/docs/bench.md +++ b/docs/bench.md @@ -233,6 +233,7 @@ Compile-only checks: ```sh cargo bench -p signinum-j2k --bench public_api --no-run +cargo bench -p signinum-j2k-native --bench tier1_bitplane --no-run cargo bench -p signinum --bench facade --no-run cargo bench -p signinum-jpeg --no-run ``` @@ -255,6 +256,18 @@ Run the J2K public API benches: cargo bench -p signinum-j2k --bench public_api ``` +Run the CPU J2K regression guard after creating a benchmark-only baseline tag: + +```sh +git tag j2k-bench-original +cargo xtask j2k-perf-guard --baseline-ref j2k-bench-original --threshold-percent 10 --quick +``` + +The guard runs deterministic Cargo Criterion benches for `signinum-j2k` and +`signinum-j2k-native` in separate baseline/current target directories, compares +median point estimates, and fails when a current result is more than 10% slower +than the baseline. External WSI inputs, Metal, and CUDA stay report-only. + Run the facade dispatch benches: ```sh diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 7973d8cf..91c8538e 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -5,6 +5,9 @@ edition = "2021" license.workspace = true publish = false +[dependencies] +serde_json = "1" + [lints.rust] unsafe_code = "forbid" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 6dc39785..32b3fa81 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,6 +2,8 @@ use std::env; use std::ffi::OsString; use std::process::{Command, ExitCode}; +mod perf_guard; + const PUBLISHABLE_PACKAGES: &[&str] = &[ "signinum-core", "signinum-cuda-runtime", @@ -64,6 +66,7 @@ fn run() -> Result<(), String> { "doc" | "docs" => doc(), "typos" => typos(), "bench-build" => bench_build(), + "j2k-perf-guard" => perf_guard::j2k_perf_guard(env::args().skip(2)), "fuzz-build" => fuzz_build(), "deny" => deny(), "no-std" => no_std(), @@ -178,6 +181,14 @@ fn bench_build() -> Result<(), String> { "public_api", "--no-run", ])?; + run_cargo(&[ + "bench", + "-p", + "signinum-j2k-native", + "--bench", + "tier1_bitplane", + "--no-run", + ])?; run_cargo(&["bench", "-p", "signinum", "--bench", "facade", "--no-run"])?; run_cargo(&["bench", "-p", "signinum-jpeg", "--no-run"])?; run_cargo(&["bench", "-p", "signinum-jpeg-metal", "--no-run"])?; @@ -375,6 +386,7 @@ fn print_help() { doc build workspace docs with warnings denied\n\ typos run typos\n\ bench-build compile benchmark targets\n\ + j2k-perf-guard compare CPU J2K Criterion medians against a baseline git ref\n\ fuzz-build compile fuzz harnesses\n\ deny run cargo-deny\n\ no-std check no_std-compatible codec crates\n\ @@ -384,3 +396,107 @@ fn print_help() { package preflight publishable package contents from a clean worktree; strict for registry-independent crates and list-only for staged dependencies" ); } + +#[cfg(test)] +mod tests { + use super::perf_guard::{ + compare_estimates, discover_estimates, BenchEstimate, RegressionOutcome, + }; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn compare_estimates_flags_median_regression_above_threshold() { + let baseline = BenchEstimate { + id: "j2k_public_decode/rgb8_full_128x128".to_string(), + median_ns: 1_000.0, + }; + let current = BenchEstimate { + id: baseline.id.clone(), + median_ns: 1_120.0, + }; + + let outcomes = compare_estimates(&[baseline], &[current], 10.0).unwrap(); + + assert_eq!( + outcomes, + vec![RegressionOutcome { + id: "j2k_public_decode/rgb8_full_128x128".to_string(), + baseline_ns: 1_000.0, + current_ns: 1_120.0, + delta_percent: 12.0, + regressed: true, + }] + ); + } + + #[test] + fn compare_estimates_allows_median_delta_at_threshold() { + let baseline = BenchEstimate { + id: "tier1_bitplane_decode/decode_64x64/default".to_string(), + median_ns: 2_000.0, + }; + let current = BenchEstimate { + id: baseline.id.clone(), + median_ns: 2_200.0, + }; + + let outcomes = compare_estimates(&[baseline], &[current], 10.0).unwrap(); + + assert!(!outcomes[0].regressed); + assert_eq!(outcomes[0].delta_percent, 10.0); + } + + #[test] + fn compare_estimates_reports_missing_current_result() { + let baseline = BenchEstimate { + id: "j2k_public_decode_gray/gray8_full_128x128".to_string(), + median_ns: 500.0, + }; + + let err = compare_estimates(&[baseline], &[], 10.0).unwrap_err(); + + assert!( + err.contains("missing current benchmark result"), + "unexpected error: {err}" + ); + } + + #[test] + fn discover_estimates_reads_criterion_median_point_estimates() { + let root = temp_dir("signinum-perf-guard-test"); + let estimate_path = root + .join("target") + .join("criterion") + .join("j2k_public_decode") + .join("rgb8_full_128x128") + .join("new"); + fs::create_dir_all(&estimate_path).unwrap(); + fs::write( + estimate_path.join("estimates.json"), + r#"{"median":{"point_estimate":321.5}}"#, + ) + .unwrap(); + + let estimates = discover_estimates(&root.join("target").join("criterion")).unwrap(); + + assert_eq!( + estimates, + vec![BenchEstimate { + id: "j2k_public_decode/rgb8_full_128x128".to_string(), + median_ns: 321.5, + }] + ); + let _ = fs::remove_dir_all(root); + } + + fn temp_dir(name: &str) -> std::path::PathBuf { + let mut dir = std::env::temp_dir(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + dir.push(format!("{name}-{}-{nanos}", std::process::id())); + dir + } +} diff --git a/xtask/src/perf_guard.rs b/xtask/src/perf_guard.rs new file mode 100644 index 00000000..fef72ac3 --- /dev/null +++ b/xtask/src/perf_guard.rs @@ -0,0 +1,358 @@ +use std::{ + collections::BTreeMap, + env, + ffi::OsString, + fs, + path::{Path, PathBuf}, + process::Command, +}; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct BenchEstimate { + pub(crate) id: String, + pub(crate) median_ns: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct RegressionOutcome { + pub(crate) id: String, + pub(crate) baseline_ns: f64, + pub(crate) current_ns: f64, + pub(crate) delta_percent: f64, + pub(crate) regressed: bool, +} + +#[derive(Debug, Clone)] +struct PerfGuardOptions { + baseline_ref: String, + threshold_percent: f64, + quick: bool, +} + +#[derive(Debug, Clone, Copy)] +struct BenchCommand { + package: &'static str, + bench: &'static str, +} + +const DEFAULT_BASELINE_REF: &str = "j2k-bench-original"; +const DEFAULT_THRESHOLD_PERCENT: f64 = 10.0; +const BENCH_COMMANDS: &[BenchCommand] = &[ + BenchCommand { + package: "signinum-j2k", + bench: "public_api", + }, + BenchCommand { + package: "signinum-j2k-native", + bench: "tier1_bitplane", + }, +]; + +pub(crate) fn j2k_perf_guard(args: impl Iterator) -> Result<(), String> { + let options = PerfGuardOptions::parse(args)?; + let root = repo_root()?; + let perf_root = root.join("target").join("signinum-perf"); + fs::create_dir_all(&perf_root) + .map_err(|err| format!("failed to create {}: {err}", perf_root.display()))?; + + let baseline_worktree = perf_root.join("baseline-worktree"); + recreate_baseline_worktree(&root, &baseline_worktree, &options.baseline_ref)?; + + let baseline_target = perf_root.join("baseline-target"); + let current_target = perf_root.join("current-target"); + reset_dir(&baseline_target)?; + reset_dir(¤t_target)?; + + run_benches(&baseline_worktree, &baseline_target, options.quick)?; + run_benches(&root, ¤t_target, options.quick)?; + + let baseline = discover_estimates(&baseline_target.join("criterion"))?; + let current = discover_estimates(¤t_target.join("criterion"))?; + let outcomes = compare_estimates(&baseline, ¤t, options.threshold_percent)?; + emit_report(&outcomes, options.threshold_percent); + + if outcomes.iter().any(|outcome| outcome.regressed) { + Err("J2K performance guard found regressions".to_string()) + } else { + Ok(()) + } +} + +pub(crate) fn compare_estimates( + baseline: &[BenchEstimate], + current: &[BenchEstimate], + threshold_percent: f64, +) -> Result, String> { + let current_by_id = current + .iter() + .map(|estimate| (estimate.id.as_str(), estimate)) + .collect::>(); + let mut outcomes = Vec::with_capacity(baseline.len()); + for base in baseline { + let Some(now) = current_by_id.get(base.id.as_str()) else { + return Err(format!("missing current benchmark result for {}", base.id)); + }; + if base.median_ns <= 0.0 { + return Err(format!( + "baseline benchmark {} has non-positive median {}", + base.id, base.median_ns + )); + } + let delta_percent = ((now.median_ns - base.median_ns) / base.median_ns) * 100.0; + outcomes.push(RegressionOutcome { + id: base.id.clone(), + baseline_ns: base.median_ns, + current_ns: now.median_ns, + delta_percent, + regressed: delta_percent > threshold_percent, + }); + } + Ok(outcomes) +} + +pub(crate) fn discover_estimates(criterion_root: &Path) -> Result, String> { + let mut out = Vec::new(); + discover_estimates_inner(criterion_root, criterion_root, &mut out)?; + out.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(out) +} + +fn discover_estimates_inner( + criterion_root: &Path, + dir: &Path, + out: &mut Vec, +) -> Result<(), String> { + if !dir.exists() { + return Err(format!( + "Criterion output directory does not exist: {}", + criterion_root.display() + )); + } + for entry in + fs::read_dir(dir).map_err(|err| format!("failed to read {}: {err}", dir.display()))? + { + let entry = + entry.map_err(|err| format!("failed to read {} entry: {err}", dir.display()))?; + let path = entry.path(); + if path.is_dir() { + discover_estimates_inner(criterion_root, &path, out)?; + continue; + } + if path.file_name().and_then(|name| name.to_str()) != Some("estimates.json") { + continue; + } + if path + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + != Some("new") + { + continue; + } + let id = estimate_id(criterion_root, &path)?; + let median_ns = read_median_estimate(&path)?; + out.push(BenchEstimate { id, median_ns }); + } + Ok(()) +} + +fn estimate_id(criterion_root: &Path, estimate_path: &Path) -> Result { + let bench_path = estimate_path + .parent() + .and_then(Path::parent) + .ok_or_else(|| { + format!( + "invalid Criterion estimate path {}", + estimate_path.display() + ) + })?; + let rel = bench_path.strip_prefix(criterion_root).map_err(|err| { + format!( + "failed to strip Criterion root {} from {}: {err}", + criterion_root.display(), + bench_path.display() + ) + })?; + let mut parts = Vec::new(); + for component in rel.components() { + parts.push(component.as_os_str().to_string_lossy().into_owned()); + } + Ok(parts.join("/")) +} + +fn read_median_estimate(path: &Path) -> Result { + let data = fs::read_to_string(path) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + let value: serde_json::Value = serde_json::from_str(&data) + .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + value + .get("median") + .and_then(|median| median.get("point_estimate")) + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| format!("{} is missing median.point_estimate", path.display())) +} + +fn emit_report(outcomes: &[RegressionOutcome], threshold_percent: f64) { + eprintln!("J2K performance guard threshold: +{threshold_percent:.2}% median"); + for outcome in outcomes { + let status = if outcome.regressed { "REGRESSED" } else { "ok" }; + eprintln!( + "{status:9} {:>9.2}% baseline={:.2}ns current={:.2}ns {}", + outcome.delta_percent, outcome.baseline_ns, outcome.current_ns, outcome.id + ); + } +} + +fn run_benches(workdir: &Path, target_dir: &Path, quick: bool) -> Result<(), String> { + for bench in BENCH_COMMANDS { + let mut args = vec!["bench", "-p", bench.package, "--bench", bench.bench]; + if quick { + args.extend_from_slice(&["--", "--quick"]); + } + run_program_with_target(cargo(), &args, workdir, target_dir)?; + } + Ok(()) +} + +fn recreate_baseline_worktree( + root: &Path, + worktree: &Path, + baseline_ref: &str, +) -> Result<(), String> { + if worktree.exists() { + run_program( + OsString::from("git"), + &["worktree", "remove", "--force", path_str(worktree)?], + root, + )?; + } + run_program( + OsString::from("git"), + &[ + "worktree", + "add", + "--detach", + path_str(worktree)?, + baseline_ref, + ], + root, + ) +} + +fn reset_dir(path: &Path) -> Result<(), String> { + if path.exists() { + fs::remove_dir_all(path) + .map_err(|err| format!("failed to remove {}: {err}", path.display()))?; + } + fs::create_dir_all(path).map_err(|err| format!("failed to create {}: {err}", path.display())) +} + +fn repo_root() -> Result { + let output = Command::new("git") + .args(["rev-parse", "--show-toplevel"]) + .output() + .map_err(|err| format!("failed to start `git rev-parse --show-toplevel`: {err}"))?; + if !output.status.success() { + return Err(format!( + "`git rev-parse --show-toplevel` exited with {}", + output.status + )); + } + let path = String::from_utf8(output.stdout) + .map_err(|err| format!("git root path was not UTF-8: {err}"))?; + Ok(PathBuf::from(path.trim())) +} + +fn run_program_with_target( + program: OsString, + args: &[&str], + workdir: &Path, + target_dir: &Path, +) -> Result<(), String> { + let display = program.to_string_lossy(); + eprintln!( + "+ CARGO_TARGET_DIR={} {} {}", + target_dir.display(), + display, + args.join(" ") + ); + let status = Command::new(&program) + .args(args) + .current_dir(workdir) + .env("CARGO_TARGET_DIR", target_dir) + .status() + .map_err(|err| format!("failed to start `{display}`: {err}"))?; + if status.success() { + Ok(()) + } else { + Err(format!("`{display}` exited with {status}")) + } +} + +fn run_program(program: OsString, args: &[&str], workdir: &Path) -> Result<(), String> { + let display = program.to_string_lossy(); + eprintln!("+ {} {}", display, args.join(" ")); + let status = Command::new(&program) + .args(args) + .current_dir(workdir) + .status() + .map_err(|err| format!("failed to start `{display}`: {err}"))?; + if status.success() { + Ok(()) + } else { + Err(format!("`{display}` exited with {status}")) + } +} + +impl PerfGuardOptions { + fn parse(mut args: impl Iterator) -> Result { + let mut options = Self { + baseline_ref: DEFAULT_BASELINE_REF.to_string(), + threshold_percent: DEFAULT_THRESHOLD_PERCENT, + quick: false, + }; + while let Some(arg) = args.next() { + match arg.as_str() { + "--baseline-ref" => { + options.baseline_ref = args + .next() + .ok_or_else(|| "--baseline-ref requires a value".to_string())?; + } + "--threshold-percent" => { + let value = args + .next() + .ok_or_else(|| "--threshold-percent requires a value".to_string())?; + options.threshold_percent = value + .parse::() + .map_err(|err| format!("invalid --threshold-percent `{value}`: {err}"))?; + if options.threshold_percent < 0.0 { + return Err("--threshold-percent must be non-negative".to_string()); + } + } + "--quick" => options.quick = true, + "--help" | "-h" => return Err(help_text()), + other => { + return Err(format!( + "unknown j2k-perf-guard argument `{other}`\n{}", + help_text() + )) + } + } + } + Ok(options) + } +} + +fn help_text() -> String { + "usage: cargo xtask j2k-perf-guard [--baseline-ref REF] [--threshold-percent N] [--quick]" + .to_string() +} + +fn path_str(path: &Path) -> Result<&str, String> { + path.to_str() + .ok_or_else(|| format!("path is not valid UTF-8: {}", path.display())) +} + +fn cargo() -> OsString { + env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")) +} From 34623f74627182c1fd826482347293a2b39ca9b9 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 17 May 2026 10:22:50 -0400 Subject: [PATCH 003/271] perf: reduce J2K decode allocation overhead --- crates/signinum-j2k-native/src/lib.rs | 27 +++ .../tests/component_planes.rs | 28 +++ crates/signinum-j2k/src/backend.rs | 4 +- crates/signinum-j2k/src/decode.rs | 210 ++++++++++++++++-- crates/signinum-j2k/src/view.rs | 58 +++-- 5 files changed, 283 insertions(+), 44 deletions(-) diff --git a/crates/signinum-j2k-native/src/lib.rs b/crates/signinum-j2k-native/src/lib.rs index 269b83a5..e3c2d617 100644 --- a/crates/signinum-j2k-native/src/lib.rs +++ b/crates/signinum-j2k-native/src/lib.rs @@ -1433,6 +1433,33 @@ impl<'a> Image<'a> { }) } + /// Decode borrowed component planes for a requested region using a + /// caller-provided decoder context. + pub fn decode_region_components_with_context<'ctx>( + &self, + roi: (u32, u32, u32, u32), + decoder_context: &'ctx mut DecoderContext<'a>, + ) -> Result> { + validate_roi((self.width(), self.height()), roi)?; + let (_x, _y, width, height) = roi; + let decoded_image = self.prepare_decoded_image_with_region(decoder_context, Some(roi))?; + let planes = decoded_image + .decoded_components + .iter() + .map(|component| ComponentPlane { + samples: component.container.truncated(), + bit_depth: component.bit_depth, + }) + .collect(); + + Ok(DecodedComponents { + dimensions: (width, height), + color_space: self.color_space.clone(), + has_alpha: self.has_alpha, + planes, + }) + } + /// Decode borrowed component planes for a requested region while /// delegating code-block/transform stages through the hidden backend hook. #[doc(hidden)] diff --git a/crates/signinum-j2k-native/tests/component_planes.rs b/crates/signinum-j2k-native/tests/component_planes.rs index 6d1f3568..7f520058 100644 --- a/crates/signinum-j2k-native/tests/component_planes.rs +++ b/crates/signinum-j2k-native/tests/component_planes.rs @@ -38,3 +38,31 @@ fn decoded_components_expose_component_planes() { } assert_eq!(interleaved, bitmap.data); } + +#[test] +fn decoded_region_components_expose_cropped_component_planes() { + let bytes = fixture(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let bitmap = image + .decode_region_with_context((1, 0, 1, 2), &mut DecoderContext::default()) + .expect("bitmap"); + let planes = image + .decode_region_components_with_context((1, 0, 1, 2), &mut context) + .expect("component region decode"); + + assert_eq!(planes.dimensions(), (1, 2)); + assert_eq!(planes.planes().len(), 3); + assert!(planes + .planes() + .iter() + .all(|plane| plane.samples().len() == 2)); + + let mut interleaved = Vec::with_capacity(6); + for idx in 0..2 { + for plane in planes.planes() { + interleaved.push(plane.samples()[idx].round() as u8); + } + } + assert_eq!(interleaved, bitmap.data); +} diff --git a/crates/signinum-j2k/src/backend.rs b/crates/signinum-j2k/src/backend.rs index a1cbedae..1b8834d0 100644 --- a/crates/signinum-j2k/src/backend.rs +++ b/crates/signinum-j2k/src/backend.rs @@ -4,7 +4,9 @@ use crate::J2kError; use alloc::string::ToString; use signinum_core::{Colorspace, Info}; -pub(crate) use signinum_j2k_native::{ColorSpace, DecodeSettings, Image, RawBitmap}; +pub(crate) use signinum_j2k_native::{ + ColorSpace, DecodeSettings, DecodedComponents, Image, RawBitmap, +}; pub(crate) fn image(bytes: &[u8], settings: DecodeSettings) -> Result, J2kError> { Image::new(bytes, &settings).map_err(|err| J2kError::Backend(err.to_string())) diff --git a/crates/signinum-j2k/src/decode.rs b/crates/signinum-j2k/src/decode.rs index 14cec9f4..343c55c9 100644 --- a/crates/signinum-j2k/src/decode.rs +++ b/crates/signinum-j2k/src/decode.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -use crate::backend::{ColorSpace, DecodeSettings, Image, RawBitmap}; +use crate::backend::{ColorSpace, DecodeSettings, DecodedComponents, Image, RawBitmap}; use crate::{backend, J2kError, J2kScratchPool}; use alloc::{string::ToString, vec::Vec}; use core::convert::Infallible; @@ -129,6 +129,12 @@ pub(crate) fn decode_image_into_with_native_context<'a>( let dims = (image.width(), image.height()); match fmt { PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { + if can_decode_u8_directly(image.color_space(), image.has_alpha(), dims, stride, fmt) { + image + .decode_into(out, native_context) + .map_err(|err| J2kError::Backend(err.to_string()))?; + return Ok(()); + } let decoded = image .decode_with_context(native_context) .map_err(|err| J2kError::Backend(err.to_string()))?; @@ -163,6 +169,22 @@ pub(crate) fn decode_image_into_with_native_context<'a>( } } +fn can_decode_u8_directly( + color_space: &ColorSpace, + has_alpha: bool, + dims: (u32, u32), + stride: usize, + fmt: PixelFormat, +) -> bool { + let width = dims.0 as usize; + match (color_space, has_alpha, fmt) { + (ColorSpace::RGB, false, PixelFormat::Rgb8) => stride == width * 3, + (ColorSpace::RGB, true, PixelFormat::Rgba8) => stride == width * 4, + (ColorSpace::Gray, false, PixelFormat::Gray8) => stride == width, + _ => false, + } +} + fn decode_image_region_into( image: &Image<'_>, out: &mut [u8], @@ -182,21 +204,12 @@ pub(crate) fn decode_image_region_into_with_native_context<'a>( fmt: PixelFormat, roi: Rect, ) -> Result<(), J2kError> { - let dims = (roi.w, roi.h); match fmt { PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { - let decoded = image - .decode_region_with_context((roi.x, roi.y, roi.w, roi.h), native_context) + let components = image + .decode_region_components_with_context((roi.x, roi.y, roi.w, roi.h), native_context) .map_err(|err| J2kError::Backend(err.to_string()))?; - write_u8_output( - image.color_space(), - image.has_alpha(), - dims, - &decoded.data, - out, - stride, - fmt, - ) + write_components_u8_output(&components, out, stride, fmt) } PixelFormat::Rgb16 | PixelFormat::Gray16 => { let raw = image @@ -219,6 +232,129 @@ pub(crate) fn decode_image_region_into_with_native_context<'a>( } } +fn write_components_u8_output( + components: &DecodedComponents<'_>, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + let (width, height) = components.dimensions(); + let width = width as usize; + let height = height as usize; + let planes = components.planes(); + match ( + components.color_space(), + components.has_alpha(), + planes.len(), + fmt, + ) { + (ColorSpace::Gray, false, 1, PixelFormat::Gray8) => { + write_component_rows_u8(&planes[0], out, stride, width, height); + Ok(()) + } + (ColorSpace::RGB, false, 3, PixelFormat::Rgb8) + | (ColorSpace::RGB, true, 4, PixelFormat::Rgb8) => { + write_rgb_component_rows_u8(planes, out, stride, width, height); + Ok(()) + } + (ColorSpace::RGB, false, 3, PixelFormat::Rgba8) => { + write_rgba_component_rows_u8(planes, out, stride, width, height, true); + Ok(()) + } + (ColorSpace::RGB, true, 4, PixelFormat::Rgba8) => { + write_rgba_component_rows_u8(planes, out, stride, width, height, false); + Ok(()) + } + _ => Err(Unsupported { + what: "backend color space cannot be mapped to requested 8-bit pixel format", + } + .into()), + } +} + +fn write_component_rows_u8( + plane: &signinum_j2k_native::ComponentPlane<'_>, + out: &mut [u8], + stride: usize, + width: usize, + height: usize, +) { + for y in 0..height { + let src = &plane.samples()[y * width..(y + 1) * width]; + let dst = &mut out[y * stride..y * stride + width]; + write_samples_as_u8(src, plane.bit_depth(), dst); + } +} + +fn write_rgb_component_rows_u8( + planes: &[signinum_j2k_native::ComponentPlane<'_>], + out: &mut [u8], + stride: usize, + width: usize, + height: usize, +) { + for y in 0..height { + let row = y * width; + let dst = &mut out[y * stride..y * stride + width * 3]; + for x in 0..width { + let dst = &mut dst[x * 3..x * 3 + 3]; + for channel in 0..3 { + dst[channel] = sample_as_u8( + planes[channel].samples()[row + x], + planes[channel].bit_depth(), + ); + } + } + } +} + +fn write_rgba_component_rows_u8( + planes: &[signinum_j2k_native::ComponentPlane<'_>], + out: &mut [u8], + stride: usize, + width: usize, + height: usize, + synthesize_alpha: bool, +) { + for y in 0..height { + let row = y * width; + let dst = &mut out[y * stride..y * stride + width * 4]; + for x in 0..width { + let dst = &mut dst[x * 4..x * 4 + 4]; + for channel in 0..3 { + dst[channel] = sample_as_u8( + planes[channel].samples()[row + x], + planes[channel].bit_depth(), + ); + } + dst[3] = if synthesize_alpha { + u8::MAX + } else { + sample_as_u8(planes[3].samples()[row + x], planes[3].bit_depth()) + }; + } + } +} + +fn write_samples_as_u8(src: &[f32], bit_depth: u8, dst: &mut [u8]) { + for (sample, dst) in src.iter().zip(dst.iter_mut()) { + *dst = sample_as_u8(*sample, bit_depth); + } +} + +fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { + let rounded = sample.round(); + if bit_depth == 8 { + return rounded.clamp(0.0, f32::from(u8::MAX)) as u8; + } + let max_value = if bit_depth >= 16 { + f32::from(u16::MAX) + } else { + f32::from(((1_u16 << bit_depth) - 1).max(1)) + }; + ((rounded.clamp(0.0, max_value) / max_value) * f32::from(u8::MAX)).round() as u8 +} + pub(crate) fn validate_supported_format(fmt: PixelFormat) -> Result<(), J2kError> { if matches!(fmt, PixelFormat::Rgba16) { return Err(Unsupported { @@ -407,3 +543,51 @@ fn convert_or_copy_u16( } } } + +#[cfg(test)] +mod tests { + use super::{can_decode_u8_directly, ColorSpace, PixelFormat}; + + #[test] + fn direct_u8_decode_accepts_exact_rgb_and_gray_layouts() { + assert!(can_decode_u8_directly( + &ColorSpace::RGB, + false, + (128, 64), + 128 * 3, + PixelFormat::Rgb8 + )); + assert!(can_decode_u8_directly( + &ColorSpace::Gray, + false, + (128, 64), + 128, + PixelFormat::Gray8 + )); + } + + #[test] + fn direct_u8_decode_rejects_format_conversion_and_padded_stride() { + assert!(!can_decode_u8_directly( + &ColorSpace::RGB, + false, + (128, 64), + 128 * 4, + PixelFormat::Rgba8 + )); + assert!(!can_decode_u8_directly( + &ColorSpace::RGB, + true, + (128, 64), + 128 * 3, + PixelFormat::Rgb8 + )); + assert!(!can_decode_u8_directly( + &ColorSpace::Gray, + false, + (128, 64), + 160, + PixelFormat::Gray8 + )); + } +} diff --git a/crates/signinum-j2k/src/view.rs b/crates/signinum-j2k/src/view.rs index 78e68365..1324ce90 100644 --- a/crates/signinum-j2k/src/view.rs +++ b/crates/signinum-j2k/src/view.rs @@ -17,9 +17,9 @@ use crate::{ use alloc::vec::Vec; use core::convert::Infallible; use signinum_core::{ - CompressedPayloadKind, CompressedTransferSyntax, DecodeRowsError, DecoderContext, Downscale, - ImageCodec, ImageDecode, ImageDecodeRows, Info, PassthroughCandidate, PixelFormat, Rect, - RowSink, TileBatchDecode, + BufferError, CompressedPayloadKind, CompressedTransferSyntax, DecodeRowsError, DecoderContext, + Downscale, ImageCodec, ImageDecode, ImageDecodeRows, Info, PassthroughCandidate, PixelFormat, + Rect, RowSink, TileBatchDecode, }; /// Borrowed parse result for a JP2 or raw JPEG 2000 / HTJ2K codestream. @@ -410,21 +410,20 @@ impl<'a> ImageDecodeRows<'a, u8> for J2kDecoder<'a> { { let fmt = row_format_u8(self.info()).map_err(DecodeRowsError::Decode)?; let row_bytes = row_bytes_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; + let full_len = row_bytes + .checked_mul(self.info.dimensions.1 as usize) + .ok_or_else(|| { + DecodeRowsError::Decode(J2kError::Buffer(BufferError::SizeOverflow { + what: "J2K row decode output buffer", + })) + })?; let mut pool = J2kScratchPool::new(); - let row = pool.packed_bytes(row_bytes); - for y in 0..self.info.dimensions.1 { - self.decode_region_into_cached( - row, - row_bytes, - fmt, - Rect { - x: 0, - y, - w: self.info.dimensions.0, - h: 1, - }, - ) + let full = pool.packed_bytes(full_len); + self.decode_into_cached(full, row_bytes, fmt) .map_err(DecodeRowsError::Decode)?; + for y in 0..self.info.dimensions.1 { + let start = y as usize * row_bytes; + let row = &full[start..start + row_bytes]; sink.write_row(y, row).map_err(DecodeRowsError::Sink)?; } Ok(signinum_core::DecodeOutcome { @@ -443,22 +442,21 @@ impl<'a> ImageDecodeRows<'a, u16> for J2kDecoder<'a> { let fmt = row_format_u16(self.info()).map_err(DecodeRowsError::Decode)?; let row_bytes = row_bytes_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; let samples_per_row = row_samples_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; + let full_len = row_bytes + .checked_mul(self.info.dimensions.1 as usize) + .ok_or_else(|| { + DecodeRowsError::Decode(J2kError::Buffer(BufferError::SizeOverflow { + what: "J2K row decode output buffer", + })) + })?; let mut pool = J2kScratchPool::new(); - let (packed, row) = pool.packed_bytes_and_row_u16(row_bytes, samples_per_row); - for y in 0..self.info.dimensions.1 { - self.decode_region_into_cached( - packed, - row_bytes, - fmt, - Rect { - x: 0, - y, - w: self.info.dimensions.0, - h: 1, - }, - ) + let (packed, row) = pool.packed_bytes_and_row_u16(full_len, samples_per_row); + self.decode_into_cached(packed, row_bytes, fmt) .map_err(DecodeRowsError::Decode)?; - for (dst, src) in row.iter_mut().zip(packed.chunks_exact(2)) { + for y in 0..self.info.dimensions.1 { + let start = y as usize * row_bytes; + let packed_row = &packed[start..start + row_bytes]; + for (dst, src) in row.iter_mut().zip(packed_row.chunks_exact(2)) { *dst = u16::from_le_bytes([src[0], src[1]]); } sink.write_row(y, row).map_err(DecodeRowsError::Sink)?; From 8d0e311f81d1bf17225b92df9ef86069a0d50f94 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 17 May 2026 10:45:17 -0400 Subject: [PATCH 004/271] perf: reuse HTJ2K decode scratch buffers --- .../src/j2c/ht_block_decode.rs | 259 ++++++++++++++++-- 1 file changed, 239 insertions(+), 20 deletions(-) diff --git a/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs b/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs index 97949f40..1d1376fb 100644 --- a/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs +++ b/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs @@ -1,6 +1,5 @@ //! Scalar HTJ2K block decoding. -use alloc::vec; use alloc::vec::Vec; use super::build::CodeBlock; @@ -11,6 +10,7 @@ use crate::error::{bail, DecodingError, Result}; #[derive(Default)] pub(crate) struct HtBlockDecodeContext { coefficients: Vec, + scratch: HtBlockDecodeScratch, width: u32, height: u32, } @@ -29,6 +29,60 @@ impl HtBlockDecodeContext { } } +#[derive(Default)] +struct HtBlockDecodeScratch { + cleanup: Vec, + v_n: Vec, + sigma: Vec, + prev_row_sig: Vec, +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct HtBlockDecodeScratchCapacities { + cleanup: usize, + v_n: usize, + sigma: usize, + prev_row_sig: usize, +} + +#[cfg(test)] +impl HtBlockDecodeScratch { + fn capacities_for_test(&self) -> HtBlockDecodeScratchCapacities { + HtBlockDecodeScratchCapacities { + cleanup: self.cleanup.capacity(), + v_n: self.v_n.capacity(), + sigma: self.sigma.capacity(), + prev_row_sig: self.prev_row_sig.capacity(), + } + } + + fn poison_for_test(&mut self) { + self.cleanup.fill(u16::MAX); + self.v_n.fill(u32::MAX); + self.sigma.fill(u16::MAX); + self.prev_row_sig.fill(u16::MAX); + } +} + +fn zeroed_u16_scratch(buffer: &mut Vec, len: usize) -> &mut [u16] { + if buffer.len() < len { + buffer.resize(len, 0); + } + buffer[..len].fill(0); + + &mut buffer[..len] +} + +fn zeroed_u32_scratch(buffer: &mut Vec, len: usize) -> &mut [u32] { + if buffer.len() < len { + buffer.resize(len, 0); + } + buffer[..len].fill(0); + + &mut buffer[..len] +} + pub(crate) fn coefficient_to_i32(value: u32, k_max: u8) -> i32 { let shift = 31_u32.saturating_sub(k_max as u32); let magnitude = ((value & 0x7FFF_FFFF) >> shift) as i32; @@ -81,7 +135,7 @@ pub(crate) fn decode( } let combined = collect_code_block_data(code_block, storage)?; - decode_combined_validated( + decode_combined_validated_with_scratch( &combined, code_block.missing_bit_planes, total_bitplanes, @@ -92,6 +146,7 @@ pub(crate) fn decode( code_block.rect.width(), code_block.rect.height(), code_block.rect.width(), + &mut ctx.scratch, ) } @@ -155,6 +210,31 @@ pub(crate) fn decode_combined( stride: u32, stripe_causal: bool, decoded_data: &mut [u32], +) -> Result<()> { + let mut scratch = HtBlockDecodeScratch::default(); + decode_combined_with_scratch( + combined, + missing_bit_planes, + number_of_coding_passes, + width, + height, + stride, + stripe_causal, + decoded_data, + &mut scratch, + ) +} + +fn decode_combined_with_scratch( + combined: &CombinedCodeBlockData, + missing_bit_planes: u8, + number_of_coding_passes: u8, + width: u32, + height: u32, + stride: u32, + stripe_causal: bool, + decoded_data: &mut [u32], + scratch: &mut HtBlockDecodeScratch, ) -> Result<()> { decode_impl( &combined.data, @@ -167,24 +247,19 @@ pub(crate) fn decode_combined( height, stride, stripe_causal, + scratch, ) .ok_or(DecodingError::CodeBlockDecodeFailure.into()) } -pub(crate) fn decode_combined_validated( - combined: &CombinedCodeBlockData, +fn validate_combined_decode( missing_bit_planes: u8, total_bitplanes: u8, number_of_coding_passes: u8, - stripe_causal: bool, strict: bool, - decoded_data: &mut [u32], - width: u32, - height: u32, - stride: u32, -) -> Result<()> { +) -> Result { if total_bitplanes == 0 { - return Ok(()); + return Ok(false); } if total_bitplanes > 31 { @@ -209,7 +284,27 @@ pub(crate) fn decode_combined_validated( bail!(DecodingError::TooManyCodingPasses); } - if number_of_coding_passes == 0 || actual_bitplanes == 0 { + Ok(number_of_coding_passes != 0 && actual_bitplanes != 0) +} + +pub(crate) fn decode_combined_validated( + combined: &CombinedCodeBlockData, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, +) -> Result<()> { + if !validate_combined_decode( + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + strict, + )? { return Ok(()); } @@ -225,6 +320,41 @@ pub(crate) fn decode_combined_validated( ) } +fn decode_combined_validated_with_scratch( + combined: &CombinedCodeBlockData, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + scratch: &mut HtBlockDecodeScratch, +) -> Result<()> { + if !validate_combined_decode( + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + strict, + )? { + return Ok(()); + } + + decode_combined_with_scratch( + combined, + missing_bit_planes, + number_of_coding_passes, + width, + height, + stride, + stripe_causal, + decoded_data, + scratch, + ) +} + struct MelDecoder<'a> { data: &'a [u8], pos: usize, @@ -486,6 +616,7 @@ fn decode_impl( height: u32, stride: u32, stripe_causal: bool, + scratch_buffers: &mut HtBlockDecodeScratch, ) -> Option<()> { if num_passes > 1 && lengths2 == 0 { num_passes = 1; @@ -518,7 +649,7 @@ fn decode_impl( let quad_rows = height.div_ceil(2) as usize; let sstr = ((width + 2 + 7) & !7) as usize; - let mut scratch = vec![0u16; sstr * (quad_rows + 1)]; + let scratch = zeroed_u16_scratch(&mut scratch_buffers.cleanup, sstr * (quad_rows + 1)); let mmsbp2 = missing_msbs + 2; { @@ -660,7 +791,7 @@ fn decode_impl( { let mut magsgn = ForwardBitReader::<0xFF>::new(&coded_data[..lcup - scup]); let v_n_width = width.div_ceil(2) as usize + 2; - let mut v_n_scratch = vec![0u32; v_n_width]; + let v_n_scratch = zeroed_u32_scratch(&mut scratch_buffers.v_n, v_n_width); let mut prev_v_n = 0u32; let mut x = 0u32; let mut sp = 0usize; @@ -767,7 +898,7 @@ fn decode_impl( if num_passes > 1 { let sigma_rows = height.div_ceil(4) as usize + 1; let mstr = ((width.div_ceil(4) + 2 + 7) & !7) as usize; - let mut sigma = vec![0u16; sigma_rows * mstr]; + let sigma = zeroed_u16_scratch(&mut scratch_buffers.sigma, sigma_rows * mstr); { let mut y = 0u32; @@ -805,7 +936,10 @@ fn decode_impl( } { - let mut prev_row_sig = vec![0u16; width.div_ceil(4) as usize + 8]; + let prev_row_sig = zeroed_u16_scratch( + &mut scratch_buffers.prev_row_sig, + width.div_ceil(4) as usize + 8, + ); let mut sigprop = ForwardBitReader::<0>::new(&coded_data[lcup..lcup + len2]); for y in (0..height).step_by(4) { @@ -834,13 +968,13 @@ fn decode_impl( let idx = (x >> 2) as usize; let ps = u32::from(prev_row_sig[idx]) | (u32::from(prev_row_sig[idx + 1]) << 16); - let ns = read_u32_pair(&sigma, next_row + idx); + let ns = read_u32_pair(sigma, next_row + idx); let mut u = (ps & 0x8888_8888) >> 3; if !stripe_causal { u |= (ns & 0x1111_1111) << 3; } - let cs = read_u32_pair(&sigma, cur_row + idx); + let cs = read_u32_pair(sigma, cur_row + idx); let mut mbr = cs; mbr |= (cs & 0x7777_7777) << 1; mbr |= (cs & 0xEEEE_EEEE) >> 1; @@ -980,7 +1114,7 @@ fn decode_impl( let half = 1u32 << (p - 2); let mstr = ((width.div_ceil(4) + 2 + 7) & !7) as usize; let sigma_rows = height.div_ceil(4) as usize + 1; - let mut sigma = vec![0u16; sigma_rows * mstr]; + let sigma = zeroed_u16_scratch(&mut scratch_buffers.sigma, sigma_rows * mstr); let mut y = 0u32; while y < height { @@ -1018,7 +1152,7 @@ fn decode_impl( for i in (0..width).step_by(8) { let cwd = magref.fetch(); - let sig = read_u32_pair(&sigma, cur_sig_idx); + let sig = read_u32_pair(sigma, cur_sig_idx); cur_sig_idx += 2; let mut col_mask = 0xFu32; let mut cwd_mut = cwd; @@ -1074,6 +1208,7 @@ mod tests { assert_eq!(encoded.num_coding_passes, 1); let mut decoded = vec![0u32; original.len()]; + let mut scratch = HtBlockDecodeScratch::default(); let decoded_ok = decode_impl( &encoded.data, &mut decoded, @@ -1085,6 +1220,7 @@ mod tests { 4, 4, false, + &mut scratch, ); assert!(decoded_ok.is_some(), "encoded={:02x?}", encoded.data); @@ -1103,6 +1239,7 @@ mod tests { assert_eq!(encoded.num_coding_passes, 1); let mut decoded = vec![0u32; original.len()]; + let mut scratch = HtBlockDecodeScratch::default(); let decoded_ok = decode_impl( &encoded.data, &mut decoded, @@ -1114,6 +1251,7 @@ mod tests { 4, 4, false, + &mut scratch, ); assert!(decoded_ok.is_some(), "encoded={:02x?}", encoded.data); @@ -1123,4 +1261,85 @@ mod tests { .collect(); assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); } + + #[test] + fn scratch_resize_zeroes_existing_values_when_growing() { + let mut scratch = HtBlockDecodeScratch::default(); + + zeroed_u16_scratch(&mut scratch.cleanup, 4).fill(7); + assert_eq!(zeroed_u16_scratch(&mut scratch.cleanup, 8), &[0; 8]); + + zeroed_u32_scratch(&mut scratch.v_n, 4).fill(9); + assert_eq!(zeroed_u32_scratch(&mut scratch.v_n, 8), &[0; 8]); + } + + #[test] + fn decode_combined_validated_with_scratch_reuses_zeroed_buffers() { + let width = 16u32; + let height = 16u32; + let original: Vec = (0..(width * height)) + .map(|i| { + let value = (i as i32 % 47) - 23; + if i % 5 == 0 { + 0 + } else { + value + } + }) + .collect(); + let total_bitplanes = 6u8; + let encoded = + encode_code_block(&original, width, height, total_bitplanes).expect("encode HT block"); + let combined = CombinedCodeBlockData { + data: encoded.data.clone(), + cleanup_length: encoded.data.len() as u32, + refinement_length: 0, + }; + let mut scratch = HtBlockDecodeScratch::default(); + let mut decoded = vec![0u32; original.len()]; + + decode_combined_validated_with_scratch( + &combined, + encoded.num_zero_bitplanes, + total_bitplanes, + encoded.num_coding_passes, + false, + true, + &mut decoded, + width, + height, + width, + &mut scratch, + ) + .expect("decode HT block"); + + let first_capacities = scratch.capacities_for_test(); + assert!(first_capacities.cleanup > 0); + assert!(first_capacities.v_n > 0); + + scratch.poison_for_test(); + decoded.fill(0); + + decode_combined_validated_with_scratch( + &combined, + encoded.num_zero_bitplanes, + total_bitplanes, + encoded.num_coding_passes, + false, + true, + &mut decoded, + width, + height, + width, + &mut scratch, + ) + .expect("decode HT block after scratch poison"); + + assert_eq!(scratch.capacities_for_test(), first_capacities); + let decoded_i32: Vec = decoded + .into_iter() + .map(|value| coefficient_to_i32(value, total_bitplanes)) + .collect(); + assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); + } } From 12bf1c114bf1778886aebb302deab98282b8b7c9 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 17 May 2026 10:50:17 -0400 Subject: [PATCH 005/271] bench: add HTJ2K context decode guard --- crates/signinum-j2k/benches/public_api.rs | 20 ++++++++++++++ crates/signinum-j2k/tests/bench_harness.rs | 1 + xtask/src/main.rs | 32 +++++++++++++++++++++- xtask/src/perf_guard.rs | 29 ++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/crates/signinum-j2k/benches/public_api.rs b/crates/signinum-j2k/benches/public_api.rs index 38d7f4e7..acc45095 100644 --- a/crates/signinum-j2k/benches/public_api.rs +++ b/crates/signinum-j2k/benches/public_api.rs @@ -221,6 +221,7 @@ fn bench_rows(c: &mut Criterion) { fn bench_tile_batch(c: &mut Criterion) { let repeated = encode_gray8_codestream(TILE_SIDE, TILE_SIDE); + let ht_repeated = encode_ht_gray8_codestream(TILE_SIDE, TILE_SIDE); let mut distinct = Vec::with_capacity(BATCH_SIZE); for idx in 0..BATCH_SIZE { let mut pixels = patterned_gray8(TILE_SIDE, TILE_SIDE); @@ -256,6 +257,25 @@ fn bench_tile_batch(c: &mut Criterion) { }); }); + group.bench_function("htj2k_gray8_repeated_batch_16", |b| { + b.iter(|| { + let mut ctx = DecoderContext::::default(); + let mut pool = J2kScratchPool::new(); + for _ in 0..BATCH_SIZE { + ::decode_tile( + &mut ctx, + &mut pool, + black_box(ht_repeated.as_slice()), + &mut out, + stride, + PixelFormat::Gray8, + ) + .expect("decode repeated htj2k gray8 tile"); + } + black_box(&out); + }); + }); + group.bench_function("gray8_distinct_batch_16", |b| { b.iter(|| { let mut ctx = DecoderContext::::default(); diff --git a/crates/signinum-j2k/tests/bench_harness.rs b/crates/signinum-j2k/tests/bench_harness.rs index 37a05a11..cd565db6 100644 --- a/crates/signinum-j2k/tests/bench_harness.rs +++ b/crates/signinum-j2k/tests/bench_harness.rs @@ -12,6 +12,7 @@ fn public_api_bench_exposes_cpu_decode_regression_surface() { "gray8_rows_128x128", "gray8_repeated_batch_16", "gray8_distinct_batch_16", + "htj2k_gray8_repeated_batch_16", "htj2k_gray8_full_512x512", ] { assert!( diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 32b3fa81..f83bcce4 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -400,7 +400,8 @@ fn print_help() { #[cfg(test)] mod tests { use super::perf_guard::{ - compare_estimates, discover_estimates, BenchEstimate, RegressionOutcome, + compare_estimates, discover_estimates, sync_benchmark_sources, BenchEstimate, + RegressionOutcome, }; use std::fs; use std::time::{SystemTime, UNIX_EPOCH}; @@ -490,6 +491,35 @@ mod tests { let _ = fs::remove_dir_all(root); } + #[test] + fn sync_benchmark_sources_overlays_current_bench_files() { + let root = temp_dir("signinum-perf-guard-sync-test"); + let source = root.join("source"); + let target = root.join("target"); + let public_bench = "crates/signinum-j2k/benches/public_api.rs"; + let native_bench = "crates/signinum-j2k-native/benches/tier1_bitplane.rs"; + fs::create_dir_all(source.join("crates/signinum-j2k/benches")).unwrap(); + fs::create_dir_all(source.join("crates/signinum-j2k-native/benches")).unwrap(); + fs::create_dir_all(target.join("crates/signinum-j2k/benches")).unwrap(); + fs::create_dir_all(target.join("crates/signinum-j2k-native/benches")).unwrap(); + fs::write(source.join(public_bench), "current public bench").unwrap(); + fs::write(source.join(native_bench), "current native bench").unwrap(); + fs::write(target.join(public_bench), "old public bench").unwrap(); + fs::write(target.join(native_bench), "old native bench").unwrap(); + + sync_benchmark_sources(&source, &target).unwrap(); + + assert_eq!( + fs::read_to_string(target.join(public_bench)).unwrap(), + "current public bench" + ); + assert_eq!( + fs::read_to_string(target.join(native_bench)).unwrap(), + "current native bench" + ); + let _ = fs::remove_dir_all(root); + } + fn temp_dir(name: &str) -> std::path::PathBuf { let mut dir = std::env::temp_dir(); let nanos = SystemTime::now() diff --git a/xtask/src/perf_guard.rs b/xtask/src/perf_guard.rs index fef72ac3..72f09ebd 100644 --- a/xtask/src/perf_guard.rs +++ b/xtask/src/perf_guard.rs @@ -47,6 +47,10 @@ const BENCH_COMMANDS: &[BenchCommand] = &[ bench: "tier1_bitplane", }, ]; +const BENCH_SOURCE_FILES: &[&str] = &[ + "crates/signinum-j2k/benches/public_api.rs", + "crates/signinum-j2k-native/benches/tier1_bitplane.rs", +]; pub(crate) fn j2k_perf_guard(args: impl Iterator) -> Result<(), String> { let options = PerfGuardOptions::parse(args)?; @@ -57,6 +61,7 @@ pub(crate) fn j2k_perf_guard(args: impl Iterator) -> Result<(), S let baseline_worktree = perf_root.join("baseline-worktree"); recreate_baseline_worktree(&root, &baseline_worktree, &options.baseline_ref)?; + sync_benchmark_sources(&root, &baseline_worktree)?; let baseline_target = perf_root.join("baseline-target"); let current_target = perf_root.join("current-target"); @@ -78,6 +83,30 @@ pub(crate) fn j2k_perf_guard(args: impl Iterator) -> Result<(), S } } +pub(crate) fn sync_benchmark_sources(source_root: &Path, target_root: &Path) -> Result<(), String> { + for relative in BENCH_SOURCE_FILES { + let source = source_root.join(relative); + let target = target_root.join(relative); + let parent = target.parent().ok_or_else(|| { + format!( + "benchmark source target has no parent directory: {}", + target.display() + ) + })?; + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + fs::copy(&source, &target).map_err(|err| { + format!( + "failed to copy benchmark source {} to {}: {err}", + source.display(), + target.display() + ) + })?; + } + + Ok(()) +} + pub(crate) fn compare_estimates( baseline: &[BenchEstimate], current: &[BenchEstimate], From c7b6f05b1e54b5c9f4d9228c3121f37ad1e9c05d Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 17 May 2026 11:12:18 -0400 Subject: [PATCH 006/271] refactor: split HTJ2K decode phases --- .../src/j2c/ht_block_decode.rs | 1173 ++++++++++------- 1 file changed, 716 insertions(+), 457 deletions(-) diff --git a/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs b/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs index 1d1376fb..71082c76 100644 --- a/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs +++ b/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs @@ -605,64 +605,121 @@ fn decode_mag_sgn_sample_with_vn( (value, v_n) } -fn decode_impl( - coded_data: &[u8], - decoded_data: &mut [u32], - missing_msbs: u32, - mut num_passes: u32, - lengths1: u32, - lengths2: u32, - width: u32, - height: u32, - stride: u32, - stripe_causal: bool, - scratch_buffers: &mut HtBlockDecodeScratch, -) -> Option<()> { - if num_passes > 1 && lengths2 == 0 { - num_passes = 1; - } +fn cleanup_symbol_stride(width: u32) -> usize { + ((width + 2 + 7) & !7) as usize +} - if num_passes > 3 || missing_msbs > 30 { - return None; - } +fn sigma_stride(width: u32) -> usize { + ((width.div_ceil(4) + 2 + 7) & !7) as usize +} - if missing_msbs == 30 { +fn cleanup_segment_suffix_length(coded_data: &[u8], lcup: usize) -> Option { + if lcup < 2 || coded_data.len() < lcup { return None; } - if missing_msbs == 29 && num_passes > 1 { - num_passes = 1; + let scup = ((coded_data[lcup - 1] as usize) << 4) + usize::from(coded_data[lcup - 2] & 0x0F); + if !(2..=lcup).contains(&scup) || scup > 4079 { + return None; } - let p = 30 - missing_msbs; - let lcup = lengths1 as usize; - let len2 = lengths2 as usize; + Some(scup) +} - if lcup < 2 || coded_data.len() < lcup.saturating_add(len2) { +#[inline(never)] +fn decode_cleanup_symbols( + coded_data: &[u8], + lcup: usize, + scup: usize, + width: u32, + height: u32, + sstr: usize, + scratch: &mut [u16], +) -> Option<()> { + let quad_rows = height.div_ceil(2) as usize; + if scratch.len() < sstr * (quad_rows + 1) { return None; } - let scup = ((coded_data[lcup - 1] as usize) << 4) + usize::from(coded_data[lcup - 2] & 0x0F); - if !(2..=lcup).contains(&scup) || scup > 4079 { - return None; - } + let mut mel = MelDecoder::new(coded_data, lcup, scup); + let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); + let mut run = mel.get_run()?; + let mut c_q = 0u32; + let mut row_offset = 0usize; + let mut x = 0u32; + + while x < width { + let mut vlc_val = vlc.fetch(); + let mut t0 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); + if c_q == 0 { + run -= 2; + t0 = if run == -1 { t0 } else { 0 }; + if run < 0 { + run = mel.get_run()?; + } + } + scratch[row_offset] = t0 as u16; + x += 2; + c_q = ((t0 & 0x10) << 3) | ((t0 & 0xE0) << 2); + vlc_val = vlc.advance(t0 & 0x7); + + let mut t1 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); + if c_q == 0 && x < width { + run -= 2; + t1 = if run == -1 { t1 } else { 0 }; + if run < 0 { + run = mel.get_run()?; + } + } + if x >= width { + t1 = 0; + } + scratch[row_offset + 2] = t1 as u16; + x += 2; + c_q = ((t1 & 0x10) << 3) | ((t1 & 0xE0) << 2); + vlc_val = vlc.advance(t1 & 0x7); + + let mut uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); + if uvlc_mode == 0xC0 { + run -= 2; + if run == -1 { + uvlc_mode += 0x40; + } + if run < 0 { + run = mel.get_run()?; + } + } - let quad_rows = height.div_ceil(2) as usize; - let sstr = ((width + 2 + 7) & !7) as usize; - let scratch = zeroed_u16_scratch(&mut scratch_buffers.cleanup, sstr * (quad_rows + 1)); - let mmsbp2 = missing_msbs + 2; + let mut uvlc_entry = u32::from(UVLC_TABLE0[(uvlc_mode + (vlc_val & 0x3F)) as usize]); + vlc_val = vlc.advance(uvlc_entry & 0x7); + uvlc_entry >>= 3; + let mut len = uvlc_entry & 0xF; + let tmp = vlc_val & ((1_u32 << len) - 1); + vlc_val = vlc.advance(len); + uvlc_entry >>= 4; + len = uvlc_entry & 0x7; + uvlc_entry >>= 3; + scratch[row_offset + 1] = (1 + (uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; + scratch[row_offset + 3] = (1 + (uvlc_entry >> 3) + (tmp >> len)) as u16; + + row_offset += 4; + } + scratch[row_offset] = 0; + scratch[row_offset + 1] = 0; - { - let mut mel = MelDecoder::new(coded_data, lcup, scup); - let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); - let mut run = mel.get_run()?; - let mut c_q = 0u32; - let mut row_offset = 0usize; + for y in (2..height).step_by(2) { + let row_base = (y >> 1) as usize * sstr; + let prev_base = row_base - sstr; let mut x = 0u32; + let mut c_q = 0u32; + let mut row_offset = row_base; while x < width { + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base)]) & 0xA0) << 2; + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x20) << 4; + let mut vlc_val = vlc.fetch(); - let mut t0 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); + let mut t0 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); if c_q == 0 { run -= 2; t0 = if run == -1 { t0 } else { 0 }; @@ -672,10 +729,14 @@ fn decode_impl( } scratch[row_offset] = t0 as u16; x += 2; - c_q = ((t0 & 0x10) << 3) | ((t0 & 0xE0) << 2); + + c_q = ((t0 & 0x40) << 2) | ((t0 & 0x80) << 1); + c_q |= u32::from(scratch[prev_base + (row_offset - row_base)]) & 0x80; + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0xA0) << 2; + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 4]) & 0x20) << 4; vlc_val = vlc.advance(t0 & 0x7); - let mut t1 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); + let mut t1 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); if c_q == 0 && x < width { run -= 2; t1 = if run == -1 { t1 } else { 0 }; @@ -688,21 +749,13 @@ fn decode_impl( } scratch[row_offset + 2] = t1 as u16; x += 2; - c_q = ((t1 & 0x10) << 3) | ((t1 & 0xE0) << 2); - vlc_val = vlc.advance(t1 & 0x7); - let mut uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); - if uvlc_mode == 0xC0 { - run -= 2; - if run == -1 { - uvlc_mode += 0x40; - } - if run < 0 { - run = mel.get_run()?; - } - } + c_q = ((t1 & 0x40) << 2) | ((t1 & 0x80) << 1); + c_q |= u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x80; + vlc_val = vlc.advance(t1 & 0x7); - let mut uvlc_entry = u32::from(UVLC_TABLE0[(uvlc_mode + (vlc_val & 0x3F)) as usize]); + let uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); + let mut uvlc_entry = u32::from(UVLC_TABLE1[(uvlc_mode + (vlc_val & 0x3F)) as usize]); vlc_val = vlc.advance(uvlc_entry & 0x7); uvlc_entry >>= 3; let mut len = uvlc_entry & 0xF; @@ -711,97 +764,405 @@ fn decode_impl( uvlc_entry >>= 4; len = uvlc_entry & 0x7; uvlc_entry >>= 3; - scratch[row_offset + 1] = (1 + (uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; - scratch[row_offset + 3] = (1 + (uvlc_entry >> 3) + (tmp >> len)) as u16; + scratch[row_offset + 1] = ((uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; + scratch[row_offset + 3] = ((uvlc_entry >> 3) + (tmp >> len)) as u16; row_offset += 4; } + scratch[row_offset] = 0; scratch[row_offset + 1] = 0; + } + + Some(()) +} + +#[inline(never)] +fn build_sigma_from_cleanup_phase( + cleanup: &[u16], + sigma: &mut [u16], + width: u32, + height: u32, + sstr: usize, + mstr: usize, +) -> Option<()> { + let sigma_rows = height.div_ceil(4) as usize + 1; + if sigma.len() < sigma_rows * mstr { + return None; + } + + let mut y = 0u32; + while y < height { + let sp_base = (y >> 1) as usize * sstr; + let dp_base = (y >> 2) as usize * mstr; + let mut x = 0u32; + let mut sp = sp_base; + let mut dp = dp_base; + while x < width { + let mut t0 = + ((u32::from(cleanup[sp]) & 0x30) >> 4) | ((u32::from(cleanup[sp]) & 0xC0) >> 2); + t0 |= ((u32::from(cleanup[sp + 2]) & 0x30) << 4) + | ((u32::from(cleanup[sp + 2]) & 0xC0) << 6); + let mut t1 = ((u32::from(cleanup[sp + sstr]) & 0x30) >> 2) + | (u32::from(cleanup[sp + sstr]) & 0xC0); + t1 |= ((u32::from(cleanup[sp + sstr + 2]) & 0x30) << 6) + | ((u32::from(cleanup[sp + sstr + 2]) & 0xC0) << 8); + sigma[dp] = (t0 | t1) as u16; + x += 4; + sp += 4; + dp += 1; + } + sigma[dp] = 0; + y += 4; + } + + let dp_base = (height.div_ceil(4) as usize) * mstr; + for x in 0..=width.div_ceil(4) as usize { + sigma[dp_base + x] = 0; + } + + Some(()) +} + +#[inline(never)] +fn apply_significance_propagation_phase( + coded_data: &[u8], + lcup: usize, + len2: usize, + sigma: &[u16], + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + mstr: usize, + stripe_causal: bool, + p: u32, + prev_row_sig: &mut [u16], +) -> Option<()> { + if coded_data.len() < lcup.saturating_add(len2) + || prev_row_sig.len() < width.div_ceil(4) as usize + 8 + { + return None; + } + + prev_row_sig.fill(0); + let mut sigprop = ForwardBitReader::<0>::new(&coded_data[lcup..lcup + len2]); + + for y in (0..height).step_by(4) { + let mut pattern = 0xFFFFu32; + if height - y < 4 { + pattern = 0x7777; + if height - y < 3 { + pattern = 0x3333; + if height - y < 2 { + pattern = 0x1111; + } + } + } + + let mut prev = 0u32; + let cur_row = (y >> 2) as usize * mstr; + let next_row = cur_row + mstr; + let dpp = (y * stride) as usize; + + for x in (0..width).step_by(4) { + let mut col_pattern = pattern; + let mut s = x as i32 + 4 - width as i32; + s = s.max(0); + col_pattern >>= (s * 4) as u32; + + let idx = (x >> 2) as usize; + let ps = u32::from(prev_row_sig[idx]) | (u32::from(prev_row_sig[idx + 1]) << 16); + let ns = read_u32_pair(sigma, next_row + idx); + let mut u = (ps & 0x8888_8888) >> 3; + if !stripe_causal { + u |= (ns & 0x1111_1111) << 3; + } + + let cs = read_u32_pair(sigma, cur_row + idx); + let mut mbr = cs; + mbr |= (cs & 0x7777_7777) << 1; + mbr |= (cs & 0xEEEE_EEEE) >> 1; + mbr |= u; + let t = mbr; + mbr |= t << 4; + mbr |= t >> 4; + mbr |= prev >> 12; + mbr &= col_pattern; + mbr &= !cs; + + let mut new_sig = mbr; + if new_sig != 0 { + let mut cwd = sigprop.fetch(); + let mut cnt = 0u32; + let mut col_mask = 0xFu32; + let inv_sig = !cs & col_pattern; + + for i in (0..16).step_by(4) { + if (col_mask & new_sig) == 0 { + col_mask <<= 4; + continue; + } + + let mut sample_mask = 0x1111u32 & col_mask; + if (new_sig & sample_mask) != 0 { + new_sig &= !sample_mask; + if (cwd & 1) != 0 { + let t = 0x33u32 << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; + cnt += 1; + } + + sample_mask <<= 1; + if (new_sig & sample_mask) != 0 { + new_sig &= !sample_mask; + if (cwd & 1) != 0 { + let t = 0x76u32 << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; + cnt += 1; + } - for y in (2..height).step_by(2) { - let row_base = (y >> 1) as usize * sstr; - let prev_base = row_base - sstr; - let mut x = 0u32; - let mut c_q = 0u32; - let mut row_offset = row_base; - - while x < width { - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base)]) & 0xA0) << 2; - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x20) << 4; - - let mut vlc_val = vlc.fetch(); - let mut t0 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); - if c_q == 0 { - run -= 2; - t0 = if run == -1 { t0 } else { 0 }; - if run < 0 { - run = mel.get_run()?; + sample_mask <<= 1; + if (new_sig & sample_mask) != 0 { + new_sig &= !sample_mask; + if (cwd & 1) != 0 { + let t = 0xECu32 << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; + cnt += 1; } + + sample_mask <<= 1; + if (new_sig & sample_mask) != 0 { + new_sig &= !sample_mask; + if (cwd & 1) != 0 { + let t = 0xC8u32 << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; + cnt += 1; + } + + col_mask <<= 4; } - scratch[row_offset] = t0 as u16; - x += 2; - - c_q = ((t0 & 0x40) << 2) | ((t0 & 0x80) << 1); - c_q |= u32::from(scratch[prev_base + (row_offset - row_base)]) & 0x80; - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0xA0) << 2; - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 4]) & 0x20) << 4; - vlc_val = vlc.advance(t0 & 0x7); - - let mut t1 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); - if c_q == 0 && x < width { - run -= 2; - t1 = if run == -1 { t1 } else { 0 }; - if run < 0 { - run = mel.get_run()?; + + if new_sig != 0 { + let mut dp = dpp + x as usize; + let value = 3u32 << (p - 2); + let mut col_mask = 0xFu32; + + for _ in 0..4 { + if (col_mask & new_sig) == 0 { + col_mask <<= 4; + dp += 1; + continue; + } + + let mut sample_mask = 0x1111u32 & col_mask; + if (new_sig & sample_mask) != 0 { + decoded_data[dp] = (cwd << 31) | value; + cwd >>= 1; + cnt += 1; + } + + sample_mask <<= 1; + if (new_sig & sample_mask) != 0 { + decoded_data[dp + stride as usize] = (cwd << 31) | value; + cwd >>= 1; + cnt += 1; + } + + sample_mask <<= 1; + if (new_sig & sample_mask) != 0 { + decoded_data[dp + 2 * stride as usize] = (cwd << 31) | value; + cwd >>= 1; + cnt += 1; + } + + sample_mask <<= 1; + if (new_sig & sample_mask) != 0 { + decoded_data[dp + 3 * stride as usize] = (cwd << 31) | value; + cwd >>= 1; + cnt += 1; + } + + col_mask <<= 4; + dp += 1; } } - if x >= width { - t1 = 0; + + sigprop.advance(cnt); + } + + let combined_sig = new_sig | cs; + prev_row_sig[idx] = combined_sig as u16; + if idx + 1 < prev_row_sig.len() { + prev_row_sig[idx + 1] = (combined_sig >> 16) as u16; + } + + let t = combined_sig; + let mut next_prev = combined_sig; + next_prev |= (t & 0x7777) << 1; + next_prev |= (t & 0xEEEE) >> 1; + prev = (next_prev | u) & 0xF000; + } + } + + Some(()) +} + +#[inline(never)] +fn apply_magnitude_refinement_phase( + coded_data: &[u8], + lcup: usize, + len2: usize, + sigma: &[u16], + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + mstr: usize, + p: u32, +) -> Option<()> { + if p < 2 || coded_data.len() < lcup.saturating_add(len2) { + return None; + } + + let mut magref = ReverseBitReader::new_mrp(coded_data, lcup, len2); + let half = 1u32 << (p - 2); + + for y in (0..height).step_by(4) { + let mut cur_sig_idx = (y >> 2) as usize * mstr; + let dpp = (y * stride) as usize; + + for i in (0..width).step_by(8) { + let cwd = magref.fetch(); + let sig = read_u32_pair(sigma, cur_sig_idx); + cur_sig_idx += 2; + let mut col_mask = 0xFu32; + let mut cwd_mut = cwd; + + if sig != 0 { + for j in 0..8 { + if (sig & col_mask) != 0 { + let mut dp = dpp + i as usize + j; + let mut sample_mask = 0x1111_1111u32 & col_mask; + + for _ in 0..4 { + if (sig & sample_mask) != 0 { + let mut sym = cwd_mut & 1; + sym = (1 - sym) << (p - 1); + sym |= half; + decoded_data[dp] ^= sym; + cwd_mut >>= 1; + } + sample_mask <<= 1; + dp += stride as usize; + } + } + col_mask <<= 4; } - scratch[row_offset + 2] = t1 as u16; - x += 2; - - c_q = ((t1 & 0x40) << 2) | ((t1 & 0x80) << 1); - c_q |= u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x80; - vlc_val = vlc.advance(t1 & 0x7); - - let uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); - let mut uvlc_entry = - u32::from(UVLC_TABLE1[(uvlc_mode + (vlc_val & 0x3F)) as usize]); - vlc_val = vlc.advance(uvlc_entry & 0x7); - uvlc_entry >>= 3; - let mut len = uvlc_entry & 0xF; - let tmp = vlc_val & ((1_u32 << len) - 1); - vlc_val = vlc.advance(len); - uvlc_entry >>= 4; - len = uvlc_entry & 0x7; - uvlc_entry >>= 3; - scratch[row_offset + 1] = ((uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; - scratch[row_offset + 3] = ((uvlc_entry >> 3) + (tmp >> len)) as u16; - - row_offset += 4; } - scratch[row_offset] = 0; - scratch[row_offset + 1] = 0; + magref.advance(sig.count_ones()); } } - { - let mut magsgn = ForwardBitReader::<0xFF>::new(&coded_data[..lcup - scup]); - let v_n_width = width.div_ceil(2) as usize + 2; - let v_n_scratch = zeroed_u32_scratch(&mut scratch_buffers.v_n, v_n_width); + Some(()) +} + +#[inline(never)] +fn decode_magnitude_sign_phase( + coded_data: &[u8], + lcup: usize, + scup: usize, + scratch: &[u16], + decoded_data: &mut [u32], + missing_msbs: u32, + width: u32, + height: u32, + stride: u32, + sstr: usize, + v_n_scratch: &mut [u32], +) -> Option<()> { + let v_n_width = width.div_ceil(2) as usize + 2; + if v_n_scratch.len() < v_n_width { + return None; + } + v_n_scratch[..v_n_width].fill(0); + + let p = 30 - missing_msbs; + let mmsbp2 = missing_msbs + 2; + let mut magsgn = ForwardBitReader::<0xFF>::new(&coded_data[..lcup - scup]); + let mut prev_v_n = 0u32; + let mut x = 0u32; + let mut sp = 0usize; + let mut vp = 0usize; + let mut dp = 0usize; + let second_row_present = height > 1; + + while x < width { + let inf = u32::from(scratch[sp]); + let uq = u32::from(scratch[sp + 1]); + if uq > mmsbp2 { + return None; + } + + let (val0, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 0, uq, p); + decoded_data[dp] = val0; + + let (val1, v_n1) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 1, uq, p); + if second_row_present { + decoded_data[dp + stride as usize] = val1; + } + v_n_scratch[vp] = prev_v_n | v_n1; + prev_v_n = 0; + dp += 1; + x += 1; + + if x >= width { + vp += 1; + break; + } + + let (val2, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 2, uq, p); + decoded_data[dp] = val2; + + let (val3, v_n3) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 3, uq, p); + if second_row_present { + decoded_data[dp + stride as usize] = val3; + } + prev_v_n = v_n3; + dp += 1; + x += 1; + sp += 2; + vp += 1; + } + v_n_scratch[vp] = prev_v_n; + + for y in (2..height).step_by(2) { + let row_base = (y >> 1) as usize * sstr; + let mut sp = row_base; + let mut vp = 0usize; + let mut dp = (y * stride) as usize; let mut prev_v_n = 0u32; let mut x = 0u32; - let mut sp = 0usize; - let mut vp = 0usize; - let mut dp = 0usize; - let second_row_present = height > 1; + let second_row_present = y + 1 < height; while x < width { let inf = u32::from(scratch[sp]); - let uq = u32::from(scratch[sp + 1]); + let u_q = u32::from(scratch[sp + 1]); + let mut gamma = inf & 0xF0; + gamma &= gamma.wrapping_sub(0x10); + let mut emax = v_n_scratch[vp] | v_n_scratch[vp + 1]; + emax = 31 - (emax | 2).leading_zeros(); + let kappa = if gamma != 0 { emax } else { 1 }; + let uq = u_q + kappa; if uq > mmsbp2 { return None; } @@ -836,352 +1197,111 @@ fn decode_impl( sp += 2; vp += 1; } - v_n_scratch[vp] = prev_v_n; - - for y in (2..height).step_by(2) { - let row_base = (y >> 1) as usize * sstr; - let mut sp = row_base; - let mut vp = 0usize; - let mut dp = (y * stride) as usize; - let mut prev_v_n = 0u32; - let mut x = 0u32; - let second_row_present = y + 1 < height; - - while x < width { - let inf = u32::from(scratch[sp]); - let u_q = u32::from(scratch[sp + 1]); - let mut gamma = inf & 0xF0; - gamma &= gamma.wrapping_sub(0x10); - let mut emax = v_n_scratch[vp] | v_n_scratch[vp + 1]; - emax = 31 - (emax | 2).leading_zeros(); - let kappa = if gamma != 0 { emax } else { 1 }; - let uq = u_q + kappa; - if uq > mmsbp2 { - return None; - } - - let (val0, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 0, uq, p); - decoded_data[dp] = val0; - - let (val1, v_n1) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 1, uq, p); - if second_row_present { - decoded_data[dp + stride as usize] = val1; - } - v_n_scratch[vp] = prev_v_n | v_n1; - prev_v_n = 0; - dp += 1; - x += 1; - - if x >= width { - vp += 1; - break; - } - - let (val2, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 2, uq, p); - decoded_data[dp] = val2; - let (val3, v_n3) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 3, uq, p); - if second_row_present { - decoded_data[dp + stride as usize] = val3; - } - prev_v_n = v_n3; - dp += 1; - x += 1; - sp += 2; - vp += 1; - } - - v_n_scratch[vp] = prev_v_n; - } + v_n_scratch[vp] = prev_v_n; } - if num_passes > 1 { - let sigma_rows = height.div_ceil(4) as usize + 1; - let mstr = ((width.div_ceil(4) + 2 + 7) & !7) as usize; - let sigma = zeroed_u16_scratch(&mut scratch_buffers.sigma, sigma_rows * mstr); - - { - let mut y = 0u32; - while y < height { - let sp_base = (y >> 1) as usize * sstr; - let dp_base = (y >> 2) as usize * mstr; - let mut x = 0u32; - let mut sp = sp_base; - let mut dp = dp_base; - - while x < width { - let mut t0 = ((u32::from(scratch[sp]) & 0x30) >> 4) - | ((u32::from(scratch[sp]) & 0xC0) >> 2); - t0 |= ((u32::from(scratch[sp + 2]) & 0x30) << 4) - | ((u32::from(scratch[sp + 2]) & 0xC0) << 6); - let mut t1 = ((u32::from(scratch[sp + sstr]) & 0x30) >> 2) - | (u32::from(scratch[sp + sstr]) & 0xC0); - t1 |= ((u32::from(scratch[sp + sstr + 2]) & 0x30) << 6) - | ((u32::from(scratch[sp + sstr + 2]) & 0xC0) << 8); - sigma[dp] = (t0 | t1) as u16; - - x += 4; - sp += 4; - dp += 1; - } - - sigma[dp] = 0; - y += 4; - } - - let dp_base = (height.div_ceil(4) as usize) * mstr; - for x in 0..=width.div_ceil(4) as usize { - sigma[dp_base + x] = 0; - } - } - - { - let prev_row_sig = zeroed_u16_scratch( - &mut scratch_buffers.prev_row_sig, - width.div_ceil(4) as usize + 8, - ); - let mut sigprop = ForwardBitReader::<0>::new(&coded_data[lcup..lcup + len2]); - - for y in (0..height).step_by(4) { - let mut pattern = 0xFFFFu32; - if height - y < 4 { - pattern = 0x7777; - if height - y < 3 { - pattern = 0x3333; - if height - y < 2 { - pattern = 0x1111; - } - } - } - - let mut prev = 0u32; - let cur_row = (y >> 2) as usize * mstr; - let next_row = cur_row + mstr; - let dpp = (y * stride) as usize; - - for x in (0..width).step_by(4) { - let mut col_pattern = pattern; - let mut s = x as i32 + 4 - width as i32; - s = s.max(0); - col_pattern >>= (s * 4) as u32; - - let idx = (x >> 2) as usize; - let ps = - u32::from(prev_row_sig[idx]) | (u32::from(prev_row_sig[idx + 1]) << 16); - let ns = read_u32_pair(sigma, next_row + idx); - let mut u = (ps & 0x8888_8888) >> 3; - if !stripe_causal { - u |= (ns & 0x1111_1111) << 3; - } + Some(()) +} - let cs = read_u32_pair(sigma, cur_row + idx); - let mut mbr = cs; - mbr |= (cs & 0x7777_7777) << 1; - mbr |= (cs & 0xEEEE_EEEE) >> 1; - mbr |= u; - let t = mbr; - mbr |= t << 4; - mbr |= t >> 4; - mbr |= prev >> 12; - mbr &= col_pattern; - mbr &= !cs; - - let mut new_sig = mbr; - if new_sig != 0 { - let mut cwd = sigprop.fetch(); - let mut cnt = 0u32; - let mut col_mask = 0xFu32; - let inv_sig = !cs & col_pattern; - - for i in (0..16).step_by(4) { - if (col_mask & new_sig) == 0 { - col_mask <<= 4; - continue; - } +fn decode_impl( + coded_data: &[u8], + decoded_data: &mut [u32], + missing_msbs: u32, + mut num_passes: u32, + lengths1: u32, + lengths2: u32, + width: u32, + height: u32, + stride: u32, + stripe_causal: bool, + scratch_buffers: &mut HtBlockDecodeScratch, +) -> Option<()> { + if num_passes > 1 && lengths2 == 0 { + num_passes = 1; + } - let mut sample_mask = 0x1111u32 & col_mask; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0x33u32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } + if num_passes > 3 || missing_msbs > 30 { + return None; + } - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0x76u32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } + if missing_msbs == 30 { + return None; + } - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0xECu32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } + if missing_msbs == 29 && num_passes > 1 { + num_passes = 1; + } - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0xC8u32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } + let p = 30 - missing_msbs; + let lcup = lengths1 as usize; + let len2 = lengths2 as usize; - col_mask <<= 4; - } + if lcup < 2 || coded_data.len() < lcup.saturating_add(len2) { + return None; + } - if new_sig != 0 { - let mut dp = dpp + x as usize; - let value = 3u32 << (p - 2); - let mut col_mask = 0xFu32; - - for _ in 0..4 { - if (col_mask & new_sig) == 0 { - col_mask <<= 4; - dp += 1; - continue; - } - - let mut sample_mask = 0x1111u32 & col_mask; - if (new_sig & sample_mask) != 0 { - decoded_data[dp] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - decoded_data[dp + stride as usize] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - decoded_data[dp + 2 * stride as usize] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - decoded_data[dp + 3 * stride as usize] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - col_mask <<= 4; - dp += 1; - } - } + let scup = cleanup_segment_suffix_length(coded_data, lcup)?; - sigprop.advance(cnt); - } + let quad_rows = height.div_ceil(2) as usize; + let sstr = cleanup_symbol_stride(width); + let scratch = zeroed_u16_scratch(&mut scratch_buffers.cleanup, sstr * (quad_rows + 1)); + decode_cleanup_symbols(coded_data, lcup, scup, width, height, sstr, scratch)?; + + let v_n_width = width.div_ceil(2) as usize + 2; + let v_n_scratch = zeroed_u32_scratch(&mut scratch_buffers.v_n, v_n_width); + decode_magnitude_sign_phase( + coded_data, + lcup, + scup, + scratch, + decoded_data, + missing_msbs, + width, + height, + stride, + sstr, + v_n_scratch, + )?; - let combined_sig = new_sig | cs; - prev_row_sig[idx] = combined_sig as u16; - if idx + 1 < prev_row_sig.len() { - prev_row_sig[idx + 1] = (combined_sig >> 16) as u16; - } + if num_passes > 1 { + let sigma_rows = height.div_ceil(4) as usize + 1; + let mstr = sigma_stride(width); + let sigma = zeroed_u16_scratch(&mut scratch_buffers.sigma, sigma_rows * mstr); + build_sigma_from_cleanup_phase(scratch, sigma, width, height, sstr, mstr)?; - let t = combined_sig; - let mut next_prev = combined_sig; - next_prev |= (t & 0x7777) << 1; - next_prev |= (t & 0xEEEE) >> 1; - prev = (next_prev | u) & 0xF000; - } - } - } + let prev_row_sig = zeroed_u16_scratch( + &mut scratch_buffers.prev_row_sig, + width.div_ceil(4) as usize + 8, + ); + apply_significance_propagation_phase( + coded_data, + lcup, + len2, + sigma, + decoded_data, + width, + height, + stride, + mstr, + stripe_causal, + p, + prev_row_sig, + )?; if num_passes > 2 { - let mut magref = ReverseBitReader::new_mrp(coded_data, lcup, len2); - let half = 1u32 << (p - 2); - let mstr = ((width.div_ceil(4) + 2 + 7) & !7) as usize; - let sigma_rows = height.div_ceil(4) as usize + 1; - let sigma = zeroed_u16_scratch(&mut scratch_buffers.sigma, sigma_rows * mstr); - - let mut y = 0u32; - while y < height { - let sp_base = (y >> 1) as usize * sstr; - let dp_base = (y >> 2) as usize * mstr; - let mut x = 0u32; - let mut sp = sp_base; - let mut dp = dp_base; - while x < width { - let mut t0 = ((u32::from(scratch[sp]) & 0x30) >> 4) - | ((u32::from(scratch[sp]) & 0xC0) >> 2); - t0 |= ((u32::from(scratch[sp + 2]) & 0x30) << 4) - | ((u32::from(scratch[sp + 2]) & 0xC0) << 6); - let mut t1 = ((u32::from(scratch[sp + sstr]) & 0x30) >> 2) - | (u32::from(scratch[sp + sstr]) & 0xC0); - t1 |= ((u32::from(scratch[sp + sstr + 2]) & 0x30) << 6) - | ((u32::from(scratch[sp + sstr + 2]) & 0xC0) << 8); - sigma[dp] = (t0 | t1) as u16; - x += 4; - sp += 4; - dp += 1; - } - sigma[dp] = 0; - y += 4; - } - - let dp_base = (height.div_ceil(4) as usize) * mstr; - for x in 0..=width.div_ceil(4) as usize { - sigma[dp_base + x] = 0; - } - - for y in (0..height).step_by(4) { - let mut cur_sig_idx = (y >> 2) as usize * mstr; - let dpp = (y * stride) as usize; - - for i in (0..width).step_by(8) { - let cwd = magref.fetch(); - let sig = read_u32_pair(sigma, cur_sig_idx); - cur_sig_idx += 2; - let mut col_mask = 0xFu32; - let mut cwd_mut = cwd; - - if sig != 0 { - for j in 0..8 { - if (sig & col_mask) != 0 { - let mut dp = dpp + i as usize + j; - let mut sample_mask = 0x1111_1111u32 & col_mask; - - for _ in 0..4 { - if (sig & sample_mask) != 0 { - let mut sym = cwd_mut & 1; - sym = (1 - sym) << (p - 1); - sym |= half; - decoded_data[dp] ^= sym; - cwd_mut >>= 1; - } - sample_mask <<= 1; - dp += stride as usize; - } - } - col_mask <<= 4; - } - } - - magref.advance(sig.count_ones()); - } - } + apply_magnitude_refinement_phase( + coded_data, + lcup, + len2, + sigma, + decoded_data, + width, + height, + stride, + mstr, + p, + )?; } } @@ -1262,6 +1382,145 @@ mod tests { assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); } + #[test] + fn cleanup_and_magnitude_sign_phases_decode_odd_sized_block() { + let width = 15u32; + let height = 13u32; + let original: Vec = (0..(width * height)) + .map(|i| { + let value = (i as i32 % 61) - 30; + if i % 7 == 0 { + 0 + } else { + value + } + }) + .collect(); + let total_bitplanes = 6u8; + let encoded = + encode_code_block(&original, width, height, total_bitplanes).expect("encode HT block"); + assert_eq!(encoded.num_coding_passes, 1); + + let lcup = encoded.data.len(); + let scup = cleanup_segment_suffix_length(&encoded.data, lcup).expect("valid cleanup info"); + let sstr = cleanup_symbol_stride(width); + let quad_rows = height.div_ceil(2) as usize; + let mut cleanup = vec![0u16; sstr * (quad_rows + 1)]; + decode_cleanup_symbols(&encoded.data, lcup, scup, width, height, sstr, &mut cleanup) + .expect("decode cleanup symbols"); + + let mut decoded = vec![0u32; original.len()]; + let mut v_n_scratch = vec![0u32; width.div_ceil(2) as usize + 2]; + decode_magnitude_sign_phase( + &encoded.data, + lcup, + scup, + &cleanup, + &mut decoded, + u32::from(encoded.num_zero_bitplanes), + width, + height, + width, + sstr, + &mut v_n_scratch, + ) + .expect("decode magnitude/sign phase"); + + let decoded_i32: Vec = decoded + .into_iter() + .map(|value| coefficient_to_i32(value, total_bitplanes)) + .collect(); + assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); + } + + #[test] + fn sigma_phase_builds_masks_and_zeroes_edge_sentinels() { + let width = 7u32; + let height = 5u32; + let sstr = cleanup_symbol_stride(width); + let mstr = sigma_stride(width); + let sigma_rows = height.div_ceil(4) as usize + 1; + let mut cleanup = vec![0u16; sstr * (height.div_ceil(2) as usize + 1)]; + cleanup[0] = 0x30; + cleanup[2] = 0xC0; + cleanup[sstr] = 0xF0; + cleanup[sstr + 2] = 0x30; + cleanup[2 * sstr] = 0xC0; + cleanup[2 * sstr + 2] = 0xF0; + let mut sigma = vec![0u16; sigma_rows * mstr]; + + build_sigma_from_cleanup_phase(&cleanup, &mut sigma, width, height, sstr, mstr) + .expect("build sigma"); + + let expected_first = (((u32::from(cleanup[0]) & 0x30) >> 4) + | ((u32::from(cleanup[0]) & 0xC0) >> 2) + | ((u32::from(cleanup[2]) & 0x30) << 4) + | ((u32::from(cleanup[2]) & 0xC0) << 6) + | ((u32::from(cleanup[sstr]) & 0x30) >> 2) + | (u32::from(cleanup[sstr]) & 0xC0) + | ((u32::from(cleanup[sstr + 2]) & 0x30) << 6) + | ((u32::from(cleanup[sstr + 2]) & 0xC0) << 8)) as u16; + let expected_second = (((u32::from(cleanup[4]) & 0x30) >> 4) + | ((u32::from(cleanup[4]) & 0xC0) >> 2) + | ((u32::from(cleanup[6]) & 0x30) << 4) + | ((u32::from(cleanup[6]) & 0xC0) << 6) + | ((u32::from(cleanup[sstr + 4]) & 0x30) >> 2) + | (u32::from(cleanup[sstr + 4]) & 0xC0) + | ((u32::from(cleanup[sstr + 6]) & 0x30) << 6) + | ((u32::from(cleanup[sstr + 6]) & 0xC0) << 8)) as u16; + assert_eq!(sigma[0], expected_first); + assert_eq!(sigma[1], expected_second); + assert_eq!(sigma[2], 0); + + let bottom = height.div_ceil(4) as usize * mstr; + for x in 0..=width.div_ceil(4) as usize { + assert_eq!(sigma[bottom + x], 0); + } + } + + #[test] + fn refinement_phases_leave_output_unchanged_for_empty_sigma() { + let width = 7u32; + let height = 5u32; + let stride = width; + let mstr = sigma_stride(width); + let sigma = vec![0u16; (height.div_ceil(4) as usize + 1) * mstr]; + let mut prev_row_sig = vec![0u16; width.div_ceil(4) as usize + 8]; + let mut decoded = vec![0x1234_5678u32; (stride * height) as usize]; + let expected = decoded.clone(); + + apply_significance_propagation_phase( + &[], + 0, + 0, + &sigma, + &mut decoded, + width, + height, + stride, + mstr, + false, + 5, + &mut prev_row_sig, + ) + .expect("empty sigma sigprop"); + apply_magnitude_refinement_phase( + &[], + 0, + 0, + &sigma, + &mut decoded, + width, + height, + stride, + mstr, + 5, + ) + .expect("empty sigma magref"); + + assert_eq!(decoded, expected); + } + #[test] fn scratch_resize_zeroes_existing_values_when_growing() { let mut scratch = HtBlockDecodeScratch::default(); From 2f8619c848a3e8af760e386baa4564776d9fa1e5 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 17 May 2026 11:20:48 -0400 Subject: [PATCH 007/271] chore: add stricter Rust hygiene xtasks --- crates/signinum-core/tests/repo_integrity.rs | 33 +++++++++++++++ xtask/src/main.rs | 42 ++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/crates/signinum-core/tests/repo_integrity.rs b/crates/signinum-core/tests/repo_integrity.rs index a9c1ebbc..72d02682 100644 --- a/crates/signinum-core/tests/repo_integrity.rs +++ b/crates/signinum-core/tests/repo_integrity.rs @@ -297,6 +297,39 @@ fn xtask_test_does_not_run_benchmarks_as_tests() { ); } +#[test] +fn xtask_exposes_nextest_machete_and_strict_clippy_gates() { + let xtask = fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask"); + let help_section = xtask + .split("fn print_help()") + .nth(1) + .expect("xtask help section"); + + for task in ["nextest", "machete", "clippy-strict"] { + assert!( + xtask.contains(&format!("\"{task}\" =>")), + "xtask must dispatch `{task}`" + ); + assert!( + help_section.contains(task), + "xtask help must document `{task}`" + ); + } + + for required in [ + "\"nextest\"", + "\"run\"", + "\"cargo-machete\"", + "\"--no-deps\"", + "\"clippy::pedantic\"", + "\"clippy::nursery\"", + "\"signinum-j2k-native\"", + "\"signinum-j2k\"", + ] { + assert!(xtask.contains(required), "xtask must contain `{required}`"); + } +} + #[test] fn xtask_fuzz_build_checks_every_fuzz_manifest() { let root = repo_root(); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index f83bcce4..2a44893a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -62,13 +62,16 @@ fn run() -> Result<(), String> { match task.as_str() { "fmt" => fmt(), "clippy" => clippy(), + "clippy-strict" => clippy_strict(), "test" => test(), + "nextest" => nextest(), "doc" | "docs" => doc(), "typos" => typos(), "bench-build" => bench_build(), "j2k-perf-guard" => perf_guard::j2k_perf_guard(env::args().skip(2)), "fuzz-build" => fuzz_build(), "deny" => deny(), + "machete" => machete(), "no-std" => no_std(), "release-cpu" => release_cpu(), "release-metal" => release_metal(), @@ -106,6 +109,26 @@ fn clippy() -> Result<(), String> { ]) } +fn clippy_strict() -> Result<(), String> { + run_cargo(&[ + "clippy", + "-p", + "signinum-j2k-native", + "-p", + "signinum-j2k", + "--all-targets", + "--all-features", + "--no-deps", + "--", + "-W", + "clippy::pedantic", + "-W", + "clippy::nursery", + "-D", + "warnings", + ]) +} + fn test() -> Result<(), String> { if env::consts::OS != "macos" { return test_workspace_without_benches(&[]); @@ -161,6 +184,18 @@ fn test_package_without_benches(package: &str, no_run: bool) -> Result<(), Strin run_cargo(&["test", "-p", package, "--all-features", "--doc"]) } +fn nextest() -> Result<(), String> { + run_cargo(&[ + "nextest", + "run", + "--workspace", + "--all-features", + "--lib", + "--bins", + "--tests", + ]) +} + fn doc() -> Result<(), String> { run_cargo_with_env( &["doc", "--workspace", "--all-features", "--no-deps"], @@ -235,6 +270,10 @@ fn deny() -> Result<(), String> { run_cargo(&["deny", "check", "licenses", "advisories", "bans", "sources"]) } +fn machete() -> Result<(), String> { + run_program(OsString::from("cargo-machete"), &[], &[]) +} + fn no_std() -> Result<(), String> { run_program( OsString::from("rustup"), @@ -382,13 +421,16 @@ fn print_help() { ci fmt, clippy, test, and docs\n\ fmt check rustfmt\n\ clippy run clippy with warnings denied\n\ + clippy-strict run stricter J2K clippy gates\n\ test run workspace tests\n\ + nextest run workspace tests with cargo-nextest\n\ doc build workspace docs with warnings denied\n\ typos run typos\n\ bench-build compile benchmark targets\n\ j2k-perf-guard compare CPU J2K Criterion medians against a baseline git ref\n\ fuzz-build compile fuzz harnesses\n\ deny run cargo-deny\n\ + machete run cargo-machete unused-dependency scan\n\ no-std check no_std-compatible codec crates\n\ release-cpu run release-mode CPU codec tests\n\ release-metal run release-mode Metal tests on macOS\n\ From 981a7aeb38c3b03becb3136aad76998af639c4a3 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 19 May 2026 09:38:48 -0400 Subject: [PATCH 008/271] perf: mature HTJ2K measured batch paths --- crates/signinum-j2k-compare/Cargo.toml | 4 + crates/signinum-j2k-compare/build.rs | 104 +- .../src/bin/jp2k_roi_batch_compare.rs | 384 +++ .../tests/bench_harness.rs | 19 + .../signinum-j2k-metal/benches/common/mod.rs | 125 +- crates/signinum-j2k-metal/benches/compare.rs | 246 +- crates/signinum-j2k-metal/src/batch.rs | 355 ++- crates/signinum-j2k-metal/src/compute.rs | 2740 +++++++++++++++-- crates/signinum-j2k-metal/src/hybrid.rs | 376 +++ crates/signinum-j2k-metal/src/idwt.metal | 68 +- crates/signinum-j2k-metal/src/lib.rs | 191 +- crates/signinum-j2k-metal/src/store.metal | 4 +- .../signinum-j2k-metal/tests/bench_harness.rs | 89 +- crates/signinum-j2k-metal/tests/device.rs | 290 +- crates/signinum-j2k-native/Cargo.toml | 8 + .../signinum-j2k-native/benches/direct_cpu.rs | 119 + .../benches/htj2k_sigprop_phase.rs | 172 ++ .../benches/tier1_bitplane.rs | 456 ++- .../fixtures/htj2k/LICENSE.OpenHTJ2K | 29 + .../fixtures/htj2k/README.md | 22 + .../htj2k/openhtj2k_ds0_ht_09_b11.gray | Bin 0 -> 629 bytes .../htj2k/openhtj2k_ds0_ht_09_b11.j2k | Bin 0 -> 790 bytes .../htj2k/openhtj2k_ds0_ht_12_b11.gray | 1 + .../htj2k/openhtj2k_ds0_ht_12_b11.j2k | Bin 0 -> 231 bytes crates/signinum-j2k-native/src/direct_cpu.rs | 719 +++++ .../src/j2c/arithmetic_encoder.rs | 30 +- .../signinum-j2k-native/src/j2c/bitplane.rs | 630 ++-- .../src/j2c/bitplane_encode.rs | 226 +- crates/signinum-j2k-native/src/j2c/decode.rs | 104 +- crates/signinum-j2k-native/src/j2c/encode.rs | 145 +- crates/signinum-j2k-native/src/j2c/fdwt.rs | 60 + .../src/j2c/ht_block_decode.rs | 860 ++++-- .../src/j2c/ht_block_encode.rs | 988 +++++- crates/signinum-j2k-native/src/j2c/idwt.rs | 66 +- crates/signinum-j2k-native/src/j2c/mod.rs | 2 +- crates/signinum-j2k-native/src/lib.rs | 853 ++++- .../tests/bench_harness.rs | 47 +- crates/signinum-j2k/benches/public_api.rs | 481 ++- crates/signinum-j2k/src/batch.rs | 489 ++- crates/signinum-j2k/tests/batch.rs | 186 +- crates/signinum-j2k/tests/bench_harness.rs | 19 + crates/signinum/benches/facade.rs | 170 +- crates/signinum/src/lib.rs | 60 +- crates/signinum/tests/bench_harness.rs | 22 + docs/architecture.md | 31 +- docs/bench.md | 85 +- xtask/src/main.rs | 100 +- xtask/src/perf_guard.rs | 690 ++++- 48 files changed, 11681 insertions(+), 1184 deletions(-) create mode 100644 crates/signinum-j2k-compare/src/bin/jp2k_roi_batch_compare.rs create mode 100644 crates/signinum-j2k-compare/tests/bench_harness.rs create mode 100644 crates/signinum-j2k-metal/src/hybrid.rs create mode 100644 crates/signinum-j2k-native/benches/direct_cpu.rs create mode 100644 crates/signinum-j2k-native/benches/htj2k_sigprop_phase.rs create mode 100644 crates/signinum-j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K create mode 100644 crates/signinum-j2k-native/fixtures/htj2k/README.md create mode 100644 crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray create mode 100644 crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k create mode 100644 crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.gray create mode 100644 crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.j2k create mode 100644 crates/signinum-j2k-native/src/direct_cpu.rs create mode 100644 crates/signinum/tests/bench_harness.rs diff --git a/crates/signinum-j2k-compare/Cargo.toml b/crates/signinum-j2k-compare/Cargo.toml index fc70492f..5ee6b871 100644 --- a/crates/signinum-j2k-compare/Cargo.toml +++ b/crates/signinum-j2k-compare/Cargo.toml @@ -23,6 +23,10 @@ cc = "1.2" name = "jp2k_batch_compare" path = "src/bin/jp2k_batch_compare.rs" +[[bin]] +name = "jp2k_roi_batch_compare" +path = "src/bin/jp2k_roi_batch_compare.rs" + [lints.rust] unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" diff --git a/crates/signinum-j2k-compare/build.rs b/crates/signinum-j2k-compare/build.rs index 30fe5cd2..26d48d6b 100644 --- a/crates/signinum-j2k-compare/build.rs +++ b/crates/signinum-j2k-compare/build.rs @@ -1,9 +1,14 @@ -use std::path::{Path, PathBuf}; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, + process::Command, +}; fn main() { println!("cargo:rustc-check-cfg=cfg(have_grok)"); println!("cargo:rerun-if-env-changed=SIGNINUM_GROK_ROOT"); println!("cargo:rerun-if-env-changed=SIGNINUM_GROK_SOURCE"); + println!("cargo:rerun-if-env-changed=PKG_CONFIG_PATH"); println!("cargo:rerun-if-changed=src/grok_shim.c"); if let Some(config) = grok_config() { @@ -20,6 +25,11 @@ fn main() { ); println!("cargo:rustc-link-lib=dylib=grokj2k"); #[cfg(target_os = "macos")] + println!( + "cargo:rustc-link-arg=-Wl,-rpath,{}", + config.lib_dir.display() + ); + #[cfg(target_os = "macos")] println!( "cargo:rustc-link-arg=-Wl,-rpath,{}", staged_lib_dir.display() @@ -39,13 +49,7 @@ fn stage_grok_runtime(lib_dir: &Path) -> Result { PathBuf::from(std::env::var_os("OUT_DIR").ok_or_else(|| "OUT_DIR missing".to_string())?); #[cfg(target_os = "macos")] { - stage_family( - lib_dir, - &out_dir, - "libgrokj2k.20.3.0.dylib", - "libgrokj2k.1.dylib", - "libgrokj2k.dylib", - )?; + stage_grok_dylib_family(lib_dir, &out_dir)?; } #[cfg(not(target_os = "macos"))] { @@ -55,28 +59,53 @@ fn stage_grok_runtime(lib_dir: &Path) -> Result { } #[cfg(target_os = "macos")] -fn stage_family( - lib_dir: &Path, - out_dir: &Path, - real_name: &str, - compat_name: &str, - link_name: &str, -) -> Result<(), String> { - let real_src = lib_dir.join(real_name); - if !real_src.exists() { - return Err(format!("missing {real_name} in {}", lib_dir.display())); - } +fn stage_grok_dylib_family(lib_dir: &Path, out_dir: &Path) -> Result<(), String> { + let real_src = find_grok_real_dylib(lib_dir)?; + let real_name = real_src + .file_name() + .and_then(OsStr::to_str) + .ok_or_else(|| format!("invalid Grok dylib name: {}", real_src.display()))?; let real_dst = out_dir.join(real_name); if real_dst.exists() { std::fs::remove_file(&real_dst) .map_err(|err| format!("remove {}: {err}", real_dst.display()))?; } - std::fs::copy(&real_src, &real_dst).map_err(|err| format!("copy {real_name}: {err}"))?; - symlink_in_dir(real_name, &out_dir.join(compat_name))?; - symlink_in_dir(compat_name, &out_dir.join(link_name))?; + std::fs::copy(&real_src, &real_dst) + .map_err(|err| format!("copy {}: {err}", real_src.display()))?; + symlink_in_dir(real_name, &out_dir.join("libgrokj2k.1.dylib"))?; + symlink_in_dir("libgrokj2k.1.dylib", &out_dir.join("libgrokj2k.dylib"))?; Ok(()) } +#[cfg(target_os = "macos")] +fn find_grok_real_dylib(lib_dir: &Path) -> Result { + let mut candidates = std::fs::read_dir(lib_dir) + .map_err(|err| format!("read Grok lib dir {}: {err}", lib_dir.display()))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>() + .map_err(|err| format!("read Grok lib dir entry: {err}"))?; + candidates.retain(|path| { + path.file_name() + .and_then(OsStr::to_str) + .is_some_and(|name| { + name.starts_with("libgrokj2k.") + && path + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("dylib")) + && name != "libgrokj2k.dylib" + && name != "libgrokj2k.1.dylib" + && !name.contains("codec") + }) + }); + candidates.sort(); + candidates.pop().ok_or_else(|| { + format!( + "missing versioned libgrokj2k dylib in {}", + lib_dir.display() + ) + }) +} + #[cfg(target_os = "macos")] fn symlink_in_dir(target_name: &str, dst: &PathBuf) -> Result<(), String> { if dst.exists() { @@ -105,11 +134,42 @@ fn grok_config() -> Option { source_include, build_include, }) + } else { + pkg_config_grok_config() + } +} + +fn pkg_config_grok_config() -> Option { + let include_dir = pkg_config_variable("libgrokj2k", "includedir")?; + let lib_dir = pkg_config_variable("libgrokj2k", "libdir")?; + if has_grok_artifacts(&lib_dir, &include_dir, &include_dir) { + Some(GrokConfig { + lib_dir, + source_include: include_dir.clone(), + build_include: include_dir, + }) } else { None } } +fn pkg_config_variable(package: &str, variable: &str) -> Option { + let output = Command::new("pkg-config") + .args(["--variable", variable, package]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let value = String::from_utf8(output.stdout).ok()?; + let value = value.trim(); + if value.is_empty() { + None + } else { + Some(PathBuf::from(value)) + } +} + fn has_grok_artifacts(lib_dir: &Path, source_include: &Path, build_include: &Path) -> bool { let header = source_include.join("grok.h"); let config_header = build_include.join("grk_config.h"); diff --git a/crates/signinum-j2k-compare/src/bin/jp2k_roi_batch_compare.rs b/crates/signinum-j2k-compare/src/bin/jp2k_roi_batch_compare.rs new file mode 100644 index 00000000..de1c775c --- /dev/null +++ b/crates/signinum-j2k-compare/src/bin/jp2k_roi_batch_compare.rs @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::num::NonZeroUsize; +use std::time::Instant; + +use signinum_core::{tile_batch_worker_count, Downscale, PixelFormat, Rect}; +use signinum_j2k::{ + decode_tiles_region_scaled_into, encode_j2k_lossless, EncodeBackendPreference, + J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, + TileBatchOptions, TileRegionScaledDecodeJob, +}; +use signinum_j2k_compare::grok; + +const DEFAULT_REPEATS: usize = 9; +const DEFAULT_BATCH_SIZE: usize = 16; + +struct CompareCase { + name: &'static str, + bytes: Vec, + roi: Rect, + scale: Downscale, + batch_size: usize, +} + +struct Measurement { + decoder: &'static str, + case_name: &'static str, + repeats: usize, + batch_size: usize, + median_us: f64, + mean_us: f64, + tiles_per_second_median: f64, + decoded_bytes_per_repeat: usize, + samples_us: Vec, +} + +fn main() { + if let Err(error) = run() { + eprintln!("{error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + if !grok::is_available() { + return Err( + "in-process Grok is unavailable; install libgrokj2k or set SIGNINUM_GROK_SOURCE/SIGNINUM_GROK_ROOT" + .to_string(), + ); + } + + let repeats = std::env::var("SIGNINUM_J2K_ROI_COMPARE_REPEATS") + .ok() + .map(|value| parse_positive_usize(&value, "SIGNINUM_J2K_ROI_COMPARE_REPEATS")) + .transpose()? + .unwrap_or(DEFAULT_REPEATS); + let workers = std::env::var("SIGNINUM_J2K_ROI_COMPARE_THREADS") + .ok() + .map(|value| parse_positive_usize(&value, "SIGNINUM_J2K_ROI_COMPARE_THREADS")) + .transpose()? + .map(|value| NonZeroUsize::new(value).expect("positive value was validated")); + + let cases = compare_cases()?; + println!( + "repeats\t{repeats}\nworkers\t{}\ngrok_available\t{}", + workers.map_or_else(|| "auto".to_string(), |value| value.get().to_string()), + grok::is_available() + ); + println!( + "decoder\tcase\tbatch_size\trepeats\tmedian_us\tmean_us\ttiles_per_second_median\tdecoded_bytes_per_repeat\tsamples_us" + ); + + for case in &cases { + validate_case(case)?; + emit_measurement(&measure_signinum(case, repeats, workers)?); + emit_measurement(&measure_grok(case, repeats, workers)?); + } + Ok(()) +} + +fn compare_cases() -> Result, String> { + let raw_512 = encode_htj2k_rgb_codestream(512, 512)?; + let jp2_512 = wrap_codestream_jp2(&raw_512, 512, 512, 3, 8, 16); + let raw_256 = encode_htj2k_rgb_codestream(256, 256)?; + let jp2_256 = wrap_codestream_jp2(&raw_256, 256, 256, 3, 8, 16); + Ok(vec![ + CompareCase { + name: "htj2k_raw_rgb8_512_roi256_q4_repeated_batch16", + bytes: raw_512, + roi: Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }, + scale: Downscale::Quarter, + batch_size: DEFAULT_BATCH_SIZE, + }, + CompareCase { + name: "htj2k_jp2_rgb8_512_roi256_q4_repeated_batch16", + bytes: jp2_512, + roi: Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }, + scale: Downscale::Quarter, + batch_size: DEFAULT_BATCH_SIZE, + }, + CompareCase { + name: "htj2k_jp2_rgb8_256_roi128_q4_repeated_batch16", + bytes: jp2_256, + roi: Rect { + x: 64, + y: 64, + w: 128, + h: 128, + }, + scale: Downscale::Quarter, + batch_size: DEFAULT_BATCH_SIZE, + }, + ]) +} + +fn encode_htj2k_rgb_codestream(width: u32, height: u32) -> Result, String> { + let pixels = patterned_rgb8(width, height); + let samples = J2kLosslessSamples::new(&pixels, width, height, 3, 8, false) + .map_err(|error| error.to_string())?; + let options = J2kLosslessEncodeOptions { + backend: EncodeBackendPreference::CpuOnly, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + max_decomposition_levels: Some(2), + validation: J2kEncodeValidation::External, + ..J2kLosslessEncodeOptions::default() + }; + Ok(encode_j2k_lossless(samples, &options) + .map_err(|error| error.to_string())? + .codestream) +} + +fn patterned_rgb8(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + pixels.push(((x * 7 + y * 11 + 17) & 0xff) as u8); + pixels.push(((x * 13 + y * 19 + 31) & 0xff) as u8); + } + } + pixels +} + +fn validate_case(case: &CompareCase) -> Result<(), String> { + let ours = decode_signinum_once(case)?; + let theirs = decode_grok_once(case, None)?; + if ours != theirs { + return Err(format!( + "{}: signinum/Grok ROI+scale output mismatch: {} vs {} bytes", + case.name, + ours.len(), + theirs.len() + )); + } + Ok(()) +} + +fn measure_signinum( + case: &CompareCase, + repeats: usize, + workers: Option, +) -> Result { + let mut samples = Vec::with_capacity(repeats); + let mut decoded = decode_signinum_once(case)?; + std::hint::black_box(&decoded); + for _ in 0..repeats { + let started = Instant::now(); + decoded = decode_signinum_once_with_workers(case, workers)?; + samples.push(started.elapsed().as_secs_f64() * 1_000_000.0); + std::hint::black_box(&decoded); + } + Ok(measurement( + "signinum", + case, + repeats, + samples, + decoded.len(), + )) +} + +fn decode_signinum_once(case: &CompareCase) -> Result, String> { + decode_signinum_once_with_workers(case, None) +} + +fn decode_signinum_once_with_workers( + case: &CompareCase, + workers: Option, +) -> Result, String> { + let output_len = output_len(case); + let stride = output_stride(case); + let mut outputs = vec![vec![0_u8; output_len]; case.batch_size]; + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: case.bytes.as_slice(), + out: out.as_mut_slice(), + stride, + roi: case.roi, + scale: case.scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions { workers }) + .map_err(|error| error.to_string())?; + Ok(outputs.into_iter().flatten().collect()) +} + +fn measure_grok( + case: &CompareCase, + repeats: usize, + workers: Option, +) -> Result { + let mut samples = Vec::with_capacity(repeats); + let mut decoded = decode_grok_once(case, workers)?; + std::hint::black_box(&decoded); + for _ in 0..repeats { + let started = Instant::now(); + decoded = decode_grok_once(case, workers)?; + samples.push(started.elapsed().as_secs_f64() * 1_000_000.0); + std::hint::black_box(&decoded); + } + Ok(measurement("grok", case, repeats, samples, decoded.len())) +} + +fn decode_grok_once(case: &CompareCase, workers: Option) -> Result, String> { + let worker_count = tile_batch_worker_count( + case.batch_size, + TileBatchOptions { workers }, + std::thread::available_parallelism().map_or(1, NonZeroUsize::get), + ); + let chunk_size = case.batch_size.div_ceil(worker_count); + let reduce = reduce_factor(case.scale)?; + let chunks = (0..case.batch_size) + .collect::>() + .chunks(chunk_size) + .map(<[_]>::to_vec) + .collect::>(); + + let outputs = std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(chunks.len()); + for chunk in chunks { + handles.push(scope.spawn(move || { + chunk + .iter() + .map(|_| grok::decode_rgb_region_scaled(&case.bytes, case.roi, reduce)) + .collect::, _>>() + })); + } + + let mut outputs = Vec::with_capacity(case.batch_size); + for handle in handles { + match handle.join() { + Ok(Ok(mut chunk_outputs)) => outputs.append(&mut chunk_outputs), + Ok(Err(error)) => return Err(error), + Err(payload) => std::panic::resume_unwind(payload), + } + } + Ok(outputs) + })?; + Ok(outputs.into_iter().flatten().collect()) +} + +fn measurement( + decoder: &'static str, + case: &CompareCase, + repeats: usize, + samples: Vec, + decoded_bytes_per_repeat: usize, +) -> Measurement { + let median_us = median(samples.clone()); + let mean_us = samples.iter().sum::() / usize_to_f64(samples.len()); + Measurement { + decoder, + case_name: case.name, + repeats, + batch_size: case.batch_size, + median_us, + mean_us, + tiles_per_second_median: usize_to_f64(case.batch_size) / (median_us / 1_000_000.0), + decoded_bytes_per_repeat, + samples_us: samples, + } +} + +fn emit_measurement(row: &Measurement) { + let samples = row + .samples_us + .iter() + .map(|value| format!("{value:.3}")) + .collect::>() + .join(","); + println!( + "{}\t{}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{}\t{}", + row.decoder, + row.case_name, + row.batch_size, + row.repeats, + row.median_us, + row.mean_us, + row.tiles_per_second_median, + row.decoded_bytes_per_repeat, + samples + ); +} + +fn output_stride(case: &CompareCase) -> usize { + case.roi.scaled_covering(case.scale).w as usize * PixelFormat::Rgb8.bytes_per_pixel() +} + +fn output_len(case: &CompareCase) -> usize { + let scaled = case.roi.scaled_covering(case.scale); + output_stride(case) * scaled.h as usize +} + +fn reduce_factor(scale: Downscale) -> Result { + match scale { + Downscale::None => Ok(0), + Downscale::Half => Ok(1), + Downscale::Quarter => Ok(2), + Downscale::Eighth => Ok(3), + _ => Err(format!("unsupported downscale for Grok compare: {scale:?}")), + } +} + +fn median(mut samples: Vec) -> f64 { + samples.sort_by(f64::total_cmp); + samples[samples.len() / 2] +} + +fn parse_positive_usize(value: &str, label: &str) -> Result { + let parsed = value + .parse::() + .map_err(|error| format!("invalid {label} {value:?}: {error}"))?; + if parsed == 0 { + return Err(format!("{label} must be > 0")); + } + Ok(parsed) +} + +fn usize_to_f64(value: usize) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +fn wrap_codestream_jp2( + codestream: &[u8], + width: u32, + height: u32, + components: u16, + bit_depth: u8, + colorspace_enum: u32, +) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); + bytes.extend_from_slice(&[ + 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', + b' ', + ]); + + let bpc = bit_depth.saturating_sub(1); + bytes.extend_from_slice(&[ + 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', + ]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&components.to_be_bytes()); + bytes.extend_from_slice(&[bpc, 7, 0, 0]); + bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); + bytes.extend_from_slice(&colorspace_enum.to_be_bytes()); + + let len = (8 + codestream.len()) as u32; + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(b"jp2c"); + bytes.extend_from_slice(codestream); + bytes +} diff --git a/crates/signinum-j2k-compare/tests/bench_harness.rs b/crates/signinum-j2k-compare/tests/bench_harness.rs new file mode 100644 index 00000000..a623025b --- /dev/null +++ b/crates/signinum-j2k-compare/tests/bench_harness.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn roi_batch_compare_binary_exposes_grok_wsi_surfaces() { + let source = include_str!("../src/bin/jp2k_roi_batch_compare.rs"); + + for expected in [ + "htj2k_raw_rgb8_512_roi256_q4_repeated_batch16", + "htj2k_jp2_rgb8_512_roi256_q4_repeated_batch16", + "htj2k_jp2_rgb8_256_roi128_q4_repeated_batch16", + "signinum", + "grok", + ] { + assert!( + source.contains(expected), + "ROI batch compare binary is missing `{expected}`" + ); + } +} diff --git a/crates/signinum-j2k-metal/benches/common/mod.rs b/crates/signinum-j2k-metal/benches/common/mod.rs index e680d3f3..43d5c44d 100644 --- a/crates/signinum-j2k-metal/benches/common/mod.rs +++ b/crates/signinum-j2k-metal/benches/common/mod.rs @@ -9,20 +9,23 @@ use signinum_core::{ TileBatchDecodeSubmit, }; use signinum_j2k::{ - decode_tiles_into, decode_tiles_region_scaled_into, CompressedTransferSyntax, DecoderContext, - Downscale, J2kContext, J2kDecoder, J2kScratchPool, PixelFormat, Rect, TileBatchOptions, - TileDecodeJob, TileRegionScaledDecodeJob, + decode_tiles_into, decode_tiles_region_scaled_into, CompressedTransferSyntax, + CpuDecodeParallelism, DecoderContext, Downscale, J2kContext, J2kDecoder, J2kScratchPool, + PixelFormat, Rect, TileBatchOptions, TileDecodeJob, TileRegionScaledDecodeJob, }; use signinum_j2k_compare::{grok, openjpeg}; use signinum_j2k_metal::{ + benchmark_group_region_scaled_requests, benchmark_region_scaled_direct_plan_prepare, extract_dicom_encapsulated_frames_with_limit, Codec as MetalJ2kCodec, - J2kDecoder as MetalJ2kDecoder, J2kScratchPool as MetalJ2kScratchPool, MetalSession, + J2kDecoder as MetalJ2kDecoder, J2kScratchPool as MetalJ2kScratchPool, MetalBackendSession, + MetalSession, }; use signinum_j2k_native::{encode, encode_htj2k, EncodeOptions}; use std::{ collections::BTreeSet, env, fs, path::{Path, PathBuf}, + sync::Arc, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -410,20 +413,42 @@ fn external_tile_batch( fn ht_bench_inputs() -> Vec { let candidates = [ - ("htj2k_gray_1024", 1024_u32, 1024_u32), - ("htj2k_gray_512", 512_u32, 512_u32), + ( + "htj2k_gray_1024", + 1024_u32, + 1024_u32, + 1_u16, + DecodeMode::Gray8, + 17_u32, + ), + ( + "htj2k_gray_512", + 512_u32, + 512_u32, + 1_u16, + DecodeMode::Gray8, + 17_u32, + ), + ( + "htj2k_rgb_512", + 512_u32, + 512_u32, + 3_u16, + DecodeMode::Rgb8, + 16_u32, + ), ]; let mut inputs = Vec::with_capacity(candidates.len()); let mut errors = Vec::new(); - for (name, width, height) in candidates { - let pixels = ht_bench_pixels(width, height, 1); - match try_encode_ht(&pixels, width, height, 1, 8) { + for (name, width, height, components, mode, colorspace) in candidates { + let pixels = ht_bench_pixels(width, height, components as usize); + match try_encode_ht(&pixels, width, height, components as u8, 8) { Ok(codestream) => inputs.push(BenchInput { name, - bytes: wrap_codestream_jp2(&codestream, width, height, 1, 8, 17), + bytes: wrap_codestream_jp2(&codestream, width, height, components, 8, colorspace), dimensions: (width, height), - mode: DecodeMode::Gray8, + mode, is_ht: true, }), Err(error) => errors.push(format!("{name}: {error}")), @@ -461,6 +486,41 @@ pub(crate) fn signinum_inspect(bytes: &[u8]) { black_box(J2kDecoder::inspect(bytes).expect("signinum inspect")); } +pub(crate) fn benchmark_region_scaled_input_arcs( + bytes: &[u8], + count: usize, + value_equal_distinct_arcs: bool, +) -> Vec> { + if value_equal_distinct_arcs { + (0..count).map(|_| Arc::<[u8]>::from(bytes)).collect() + } else { + let input = Arc::<[u8]>::from(bytes); + vec![input; count] + } +} + +pub(crate) fn signinum_benchmark_region_scaled_direct_plan_prepare( + input: &BenchInput, + edge: u32, + scale: Downscale, +) -> bool { + let roi = centered_roi(input.dimensions, edge); + benchmark_region_scaled_direct_plan_prepare(&input.bytes, mode_format(input.mode), roi, scale) + .is_ok() +} + +pub(crate) fn signinum_benchmark_group_region_scaled_requests( + inputs: &[Arc<[u8]>], + mode: DecodeMode, + roi: Rect, + scale: Downscale, + backend: BackendRequest, +) { + let grouped = + benchmark_group_region_scaled_requests(inputs, mode_format(mode), backend, roi, scale); + black_box((grouped.batch_count, grouped.max_batch_len)); +} + pub(crate) fn signinum_decode(bytes: &[u8], mode: DecodeMode) { let mut decoder = J2kDecoder::new(bytes).expect("signinum decoder"); let info = decoder.info().dimensions; @@ -472,6 +532,18 @@ pub(crate) fn signinum_decode(bytes: &[u8], mode: DecodeMode) { black_box(out); } +pub(crate) fn signinum_decode_serial(bytes: &[u8], mode: DecodeMode) { + let mut decoder = J2kDecoder::new(bytes).expect("signinum decoder"); + decoder.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + let info = decoder.info().dimensions; + let (fmt, stride) = mode_geometry(mode, info); + let mut out = vec![0_u8; stride * info.1 as usize]; + decoder + .decode_into(&mut out, stride, fmt) + .expect("signinum serial decode"); + black_box(out); +} + pub(crate) fn signinum_decode_region(bytes: &[u8], mode: DecodeMode, edge: u32) { let mut decoder = J2kDecoder::new(bytes).expect("signinum decoder"); let roi = centered_roi(decoder.info().dimensions, edge); @@ -988,6 +1060,32 @@ pub(crate) fn signinum_metal_decode_tile_batch_region_scaled( black_box(surfaces); } +pub(crate) fn signinum_cpu_staged_metal_decode_tile_batch_region_scaled( + bytes: &[u8], + mode: DecodeMode, + edge: u32, + scale: Downscale, + count: usize, +) { + let cpu_decoder = J2kDecoder::new(bytes).expect("signinum decoder"); + let roi = centered_roi(cpu_decoder.info().dimensions, edge); + let session = MetalBackendSession::system_default().expect("Metal session"); + let surfaces = (0..count) + .map(|_| { + let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); + decoder + .decode_region_scaled_to_cpu_staged_metal_surface_with_session( + mode_format(mode), + roi, + scale, + &session, + ) + .expect("signinum CPU-staged Metal tile region scaled decode") + }) + .collect::>(); + black_box(surfaces); +} + pub(crate) fn signinum_metal_decode_tile_batch_distinct(inputs: &[Vec], mode: DecodeMode) { let mut ctx = DecoderContext::::new(); let mut session = MetalSession::default(); @@ -1553,7 +1651,10 @@ fn mode_format(mode: DecodeMode) -> PixelFormat { } fn supports_metal_region_scaled_mode(mode: DecodeMode) -> bool { - matches!(mode, DecodeMode::Gray8 | DecodeMode::Gray16) + matches!( + mode, + DecodeMode::Gray8 | DecodeMode::Gray16 | DecodeMode::Rgb8 + ) } fn mode_geometry(mode: DecodeMode, dims: (u32, u32)) -> (PixelFormat, usize) { diff --git a/crates/signinum-j2k-metal/benches/compare.rs b/crates/signinum-j2k-metal/benches/compare.rs index 45a270e1..2441d97e 100644 --- a/crates/signinum-j2k-metal/benches/compare.rs +++ b/crates/signinum-j2k-metal/benches/compare.rs @@ -3,20 +3,25 @@ mod common; use common::{ - bench_inputs, distinct_gray_tile_batch_inputs, distinct_rgb_tile_batch_inputs, - external_wsi_tile_batches, grok_decode_external_tile_batch_region_scaled, - grok_supports_external_tile_batch_region_scaled, j2k_region_edges, j2k_tile_batch_sizes, - metal_available, openjpeg_decode_external_tile_batch_region_scaled, openjpeg_decode_tile_batch, + bench_inputs, benchmark_region_scaled_input_arcs, distinct_gray_tile_batch_inputs, + distinct_rgb_tile_batch_inputs, external_wsi_tile_batches, + grok_decode_external_tile_batch_region_scaled, grok_supports_external_tile_batch_region_scaled, + j2k_region_edges, j2k_tile_batch_sizes, metal_available, + openjpeg_decode_external_tile_batch_region_scaled, openjpeg_decode_tile_batch, openjpeg_decode_tile_batch_distinct, openjpeg_supports_external_tile_batch_region_scaled, signinum_adaptive_decode, signinum_adaptive_decode_external_tile_batch_region_scaled, signinum_adaptive_decode_region, signinum_adaptive_decode_region_scaled, signinum_adaptive_decode_scaled, signinum_adaptive_decode_tile_batch, signinum_adaptive_decode_tile_batch_region_scaled, - signinum_adaptive_decode_tile_batch_region_scaled_distinct, signinum_decode, + signinum_adaptive_decode_tile_batch_region_scaled_distinct, + signinum_benchmark_group_region_scaled_requests, + signinum_benchmark_region_scaled_direct_plan_prepare, + signinum_cpu_staged_metal_decode_tile_batch_region_scaled, signinum_decode, signinum_decode_external_tile_batch_region_scaled, signinum_decode_region, - signinum_decode_region_scaled, signinum_decode_scaled, signinum_decode_tile_batch, - signinum_decode_tile_batch_distinct, signinum_decode_tile_batch_region_scaled, - signinum_decode_tile_batch_region_scaled_distinct, signinum_inspect, signinum_metal_decode, + signinum_decode_region_scaled, signinum_decode_scaled, signinum_decode_serial, + signinum_decode_tile_batch, signinum_decode_tile_batch_distinct, + signinum_decode_tile_batch_region_scaled, signinum_decode_tile_batch_region_scaled_distinct, + signinum_inspect, signinum_metal_decode, signinum_metal_decode_external_tile_batch_region_scaled, signinum_metal_decode_region, signinum_metal_decode_region_scaled, signinum_metal_decode_scaled, signinum_metal_decode_tile_batch, signinum_metal_decode_tile_batch_distinct, @@ -29,7 +34,7 @@ use common::{ signinum_metal_supports_tile_batch_region_scaled_distinct, DecodeMode, }; use criterion::{criterion_group, criterion_main, Criterion}; -use signinum_core::Rect; +use signinum_core::{BackendRequest, Rect}; use signinum_j2k::Downscale; use signinum_j2k_compare::{grok, openjpeg}; @@ -81,6 +86,9 @@ fn bench_compare(c: &mut Criterion) { decode_rgb.bench_function(format!("signinum/{}", input.name), |b| { b.iter(|| signinum_decode(&input.bytes, input.mode)); }); + decode_rgb.bench_function(format!("signinum-serial/{}", input.name), |b| { + b.iter(|| signinum_decode_serial(&input.bytes, input.mode)); + }); if !input.is_ht && openjpeg::is_available() { decode_rgb.bench_function(format!("openjpeg/{}", input.name), |b| { b.iter(|| openjpeg::decode_rgb(&input.bytes).expect("OpenJPEG decode")); @@ -301,6 +309,169 @@ fn bench_compare(c: &mut Criterion) { } wsi_tile_batch_region_scaled.finish(); + let mut wsi_tile_batch_region_scaled_rgb = + c.benchmark_group("wsi_tile_batch_region_scaled_rgb_q4"); + for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb8) { + for &count in &batch_sizes { + wsi_tile_batch_region_scaled_rgb.bench_function( + format!("signinum/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_decode_tile_batch_region_scaled( + &input.bytes, + input.mode, + 256, + Downscale::Quarter, + count, + ); + }); + }, + ); + if metal_available() { + wsi_tile_batch_region_scaled_rgb.bench_function( + format!("signinum-cpu-staged-metal/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_cpu_staged_metal_decode_tile_batch_region_scaled( + &input.bytes, + input.mode, + 256, + Downscale::Quarter, + count, + ); + }); + }, + ); + } + if metal_available() + && signinum_metal_supports_tile_batch_region_scaled( + &input.bytes, + input.mode, + 256, + Downscale::Quarter, + ) + { + wsi_tile_batch_region_scaled_rgb.bench_function( + format!("signinum-metal-resident/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_metal_decode_tile_batch_region_scaled( + &input.bytes, + input.mode, + 256, + Downscale::Quarter, + count, + ); + }); + }, + ); + } + } + } + wsi_tile_batch_region_scaled_rgb.finish(); + + let mut htj2k_plan_build = c.benchmark_group("htj2k_region_scaled_plan_build"); + for input in inputs.iter().filter(|input| input.is_ht) { + if metal_available() + && signinum_benchmark_region_scaled_direct_plan_prepare(input, 256, Downscale::Quarter) + { + htj2k_plan_build.bench_function(format!("{}/edge_256/q4", input.name), |b| { + b.iter(|| { + signinum_benchmark_region_scaled_direct_plan_prepare( + input, + 256, + Downscale::Quarter, + ); + }); + }); + } + } + htj2k_plan_build.finish(); + + let mut htj2k_feeder = c.benchmark_group("htj2k_feeder_coalesce"); + for input in inputs.iter().filter(|input| input.is_ht) { + for &count in &batch_sizes { + let roi = common::centered_roi(input.dimensions, 256); + let ptr_eq_inputs = benchmark_region_scaled_input_arcs(&input.bytes, count, false); + let value_equal_inputs = benchmark_region_scaled_input_arcs(&input.bytes, count, true); + htj2k_feeder.bench_function(format!("{}/ptr_eq/batch_{count}", input.name), |b| { + b.iter(|| { + signinum_benchmark_group_region_scaled_requests( + &ptr_eq_inputs, + input.mode, + roi, + Downscale::Quarter, + BackendRequest::Auto, + ); + }); + }); + htj2k_feeder.bench_function( + format!("{}/value_equal_arcs/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_benchmark_group_region_scaled_requests( + &value_equal_inputs, + input.mode, + roi, + Downscale::Quarter, + BackendRequest::Auto, + ); + }); + }, + ); + } + } + htj2k_feeder.finish(); + + let mut htj2k_metal_route = c.benchmark_group("htj2k_metal_route"); + for input in inputs + .iter() + .filter(|input| input.is_ht && input.mode == DecodeMode::Rgb8) + { + for &count in &batch_sizes { + if metal_available() { + htj2k_metal_route.bench_function( + format!("signinum-cpu-staged-metal/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_cpu_staged_metal_decode_tile_batch_region_scaled( + &input.bytes, + input.mode, + 256, + Downscale::Quarter, + count, + ); + }); + }, + ); + } + if metal_available() + && signinum_metal_supports_tile_batch_region_scaled( + &input.bytes, + input.mode, + 256, + Downscale::Quarter, + ) + { + htj2k_metal_route.bench_function( + format!("signinum-metal-resident/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_metal_decode_tile_batch_region_scaled( + &input.bytes, + input.mode, + 256, + Downscale::Quarter, + count, + ); + }); + }, + ); + } + } + } + htj2k_metal_route.finish(); + let mut wsi_tile_batch_region_scaled_distinct = c.benchmark_group("wsi_tile_batch_region_scaled_gray_distinct_q4"); for input in inputs @@ -361,6 +532,63 @@ fn bench_compare(c: &mut Criterion) { } wsi_tile_batch_region_scaled_distinct.finish(); + let mut wsi_tile_batch_region_scaled_rgb_distinct = + c.benchmark_group("wsi_tile_batch_region_scaled_rgb_distinct_q4"); + for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb8) { + for &count in &batch_sizes { + let distinct_inputs = distinct_rgb_tile_batch_inputs(input, count); + wsi_tile_batch_region_scaled_rgb_distinct.bench_function( + format!("signinum/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_decode_tile_batch_region_scaled_distinct( + &distinct_inputs, + input.mode, + 256, + Downscale::Quarter, + ); + }); + }, + ); + wsi_tile_batch_region_scaled_rgb_distinct.bench_function( + format!("signinum-adaptive/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_adaptive_decode_tile_batch_region_scaled_distinct( + &distinct_inputs, + input.mode, + 256, + Downscale::Quarter, + ); + }); + }, + ); + if metal_available() + && signinum_metal_supports_tile_batch_region_scaled_distinct( + &distinct_inputs, + input.mode, + 256, + Downscale::Quarter, + ) + { + wsi_tile_batch_region_scaled_rgb_distinct.bench_function( + format!("signinum-metal/{}/batch_{count}", input.name), + |b| { + b.iter(|| { + signinum_metal_decode_tile_batch_region_scaled_distinct( + &distinct_inputs, + input.mode, + 256, + Downscale::Quarter, + ); + }); + }, + ); + } + } + } + wsi_tile_batch_region_scaled_rgb_distinct.finish(); + let external_batches = external_wsi_tile_batches(max_batch_size); if !external_batches.is_empty() { let mut external_wsi_region_scaled = diff --git a/crates/signinum-j2k-metal/src/batch.rs b/crates/signinum-j2k-metal/src/batch.rs index 12fc36c2..575e4d7a 100644 --- a/crates/signinum-j2k-metal/src/batch.rs +++ b/crates/signinum-j2k-metal/src/batch.rs @@ -1,6 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -use std::sync::{Arc, Mutex}; +use std::{ + cell::OnceCell, + collections::hash_map::DefaultHasher, + hash::{Hash, Hasher}, + sync::{Arc, Mutex}, +}; use signinum_core::{BackendKind, BackendRequest, DeviceSubmission, Downscale, PixelFormat, Rect}; use signinum_j2k::{ @@ -10,10 +15,19 @@ use signinum_j2k::{ use crate::{Error, J2kDecoder, MetalSession, Storage, Surface, SurfaceResidency}; -const AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_DIM: u32 = 512; -const AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_COUNT: usize = 64; -const AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_DIM: u32 = 1024; -const AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_COUNT: usize = 16; +const AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_DIM: u32 = 512; +const AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_COUNT: usize = 64; +const AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM: u32 = 512; +const AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT: usize = 2; +const AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM: u32 = 1024; +const AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT: usize = 16; +const REGION_SCALED_DIRECT_FORMATS: [PixelFormat; 5] = [ + PixelFormat::Gray8, + PixelFormat::Gray16, + PixelFormat::Rgb8, + PixelFormat::Rgba8, + PixelFormat::Rgb16, +]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum BatchOp { @@ -30,13 +44,76 @@ struct QueuedRequest { backend: BackendRequest, op: BatchOp, output_slot: usize, + max_image_dim: OnceCell>, + input_fingerprint: OnceCell, } impl QueuedRequest { fn max_image_dim(&self) -> Option { - let decoder = J2kDecoder::new(self.input.as_ref()).ok()?; - let dims = decoder.inner.info().dimensions; - Some(dims.0.max(dims.1)) + *self.max_image_dim.get_or_init(|| { + let decoder = J2kDecoder::new(self.input.as_ref()).ok()?; + let dims = decoder.inner.info().dimensions; + Some(dims.0.max(dims.1)) + }) + } + + fn input_fingerprint(&self) -> u64 { + *self.input_fingerprint.get_or_init(|| { + let mut hasher = DefaultHasher::new(); + self.input.len().hash(&mut hasher); + if !self.input.is_empty() { + let len = self.input.len(); + for offset in [0, len / 4, len / 2, len.saturating_sub(8)] { + let end = offset.saturating_add(8).min(len); + self.input[offset..end].hash(&mut hasher); + } + } + hasher.finish() + }) + } + + #[cfg(test)] + fn max_image_dim_cache_filled_for_test(&self) -> bool { + self.max_image_dim.get().is_some() + } + + #[cfg(test)] + fn input_fingerprint_cache_filled_for_test(&self) -> bool { + self.input_fingerprint.get().is_some() + } +} + +#[doc(hidden)] +pub struct BenchmarkGroupedRequests { + pub batch_count: usize, + pub max_batch_len: usize, +} + +#[doc(hidden)] +pub fn benchmark_group_region_scaled_requests( + inputs: &[Arc<[u8]>], + fmt: PixelFormat, + backend: BackendRequest, + roi: Rect, + scale: Downscale, +) -> BenchmarkGroupedRequests { + let queued = inputs + .iter() + .enumerate() + .map(|(output_slot, input)| QueuedRequest { + input: input.clone(), + fmt, + backend, + op: BatchOp::RegionScaled { roi, scale }, + output_slot, + max_image_dim: OnceCell::new(), + input_fingerprint: OnceCell::new(), + }) + .collect::>(); + let batches = group_metal_requests(queued); + BenchmarkGroupedRequests { + batch_count: batches.len(), + max_batch_len: batches.iter().map(Vec::len).max().unwrap_or(0), } } @@ -92,6 +169,8 @@ pub(crate) fn queue_tile_request_shared( backend, op, output_slot: slot, + max_image_dim: OnceCell::new(), + input_fingerprint: OnceCell::new(), }); MetalSubmission { session: session.shared.clone(), @@ -110,7 +189,7 @@ fn flush_if_needed(session: &mut SessionState) { } fn group_metal_requests(queued: Vec) -> Vec> { - coalesce_cpu_host_batches(coalesce_distinct_region_scaled_grayscale_metal_requests( + coalesce_cpu_host_batches(coalesce_distinct_region_scaled_direct_metal_requests( coalesce_distinct_full_color_metal_requests( coalesce_distinct_full_grayscale_metal_requests(group_repeated_full_metal_requests( queued, @@ -162,37 +241,39 @@ fn coalesce_distinct_full_grayscale_metal_requests( batches } -fn coalesce_distinct_region_scaled_grayscale_metal_requests( +fn coalesce_distinct_region_scaled_direct_metal_requests( repeated_batches: Vec>, ) -> Vec> { let mut batches = Vec::new(); - let mut metal_gray8 = Vec::new(); - let mut metal_gray16 = Vec::new(); - let mut auto_gray8 = Vec::new(); - let mut auto_gray16 = Vec::new(); + let mut metal_by_format: [Vec; REGION_SCALED_DIRECT_FORMATS.len()] = + std::array::from_fn(|_| Vec::new()); + let mut auto_by_format: [Vec; REGION_SCALED_DIRECT_FORMATS.len()] = + std::array::from_fn(|_| Vec::new()); for batch in repeated_batches { - if batch.len() == 1 && is_region_scaled_grayscale_batch_candidate(&batch[0]) { + if batch.len() == 1 && is_region_scaled_direct_batch_candidate(&batch[0]) { let request = batch .into_iter() .next() .expect("single-entry batch has request"); - match (request.backend, request.fmt) { - (BackendRequest::Metal, PixelFormat::Gray8) => metal_gray8.push(request), - (BackendRequest::Metal, PixelFormat::Gray16) => metal_gray16.push(request), - (BackendRequest::Auto, PixelFormat::Gray8) => auto_gray8.push(request), - (BackendRequest::Auto, PixelFormat::Gray16) => auto_gray16.push(request), - _ => unreachable!("candidate backend and pixel format are restricted above"), + let format_idx = region_scaled_direct_format_index(request.fmt) + .expect("candidate pixel format is restricted above"); + match request.backend { + BackendRequest::Metal => metal_by_format[format_idx].push(request), + BackendRequest::Auto => auto_by_format[format_idx].push(request), + _ => unreachable!("candidate backend is restricted above"), } } else { batches.push(batch); } } - push_coalesced_or_single(&mut batches, metal_gray8); - push_coalesced_or_single(&mut batches, metal_gray16); - push_auto_region_scaled_grayscale_batches(&mut batches, auto_gray8); - push_auto_region_scaled_grayscale_batches(&mut batches, auto_gray16); + for requests in metal_by_format { + push_coalesced_or_single(&mut batches, requests); + } + for requests in auto_by_format { + push_auto_region_scaled_direct_batches(&mut batches, requests); + } batches } @@ -207,11 +288,11 @@ fn push_coalesced_or_single(batches: &mut Vec>, requests: Vec } } -fn push_auto_region_scaled_grayscale_batches( +fn push_auto_region_scaled_direct_batches( batches: &mut Vec>, requests: Vec, ) { - let Some(min_dim) = auto_region_scaled_grayscale_metal_min_dim(&requests) else { + let Some(min_dim) = auto_region_scaled_direct_metal_min_dim(&requests) else { push_coalesced_or_single(batches, requests); return; }; @@ -313,7 +394,7 @@ fn can_decode_as_repeated_full_grayscale_batch( && is_repeated_full_grayscale_candidate(next) && first.fmt == next.fmt && first.backend == next.backend - && first.input.as_ref() == next.input.as_ref() + && same_input_bytes(first, next) } fn can_decode_as_repeated_full_color_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { @@ -321,7 +402,20 @@ fn can_decode_as_repeated_full_color_batch(first: &QueuedRequest, next: &QueuedR && is_repeated_full_color_candidate(next) && first.fmt == next.fmt && first.backend == next.backend - && first.input.as_ref() == next.input.as_ref() + && same_input_bytes(first, next) +} + +fn same_input_bytes(first: &QueuedRequest, next: &QueuedRequest) -> bool { + if Arc::ptr_eq(&first.input, &next.input) { + return true; + } + if first.input.len() != next.input.len() { + return false; + } + if first.input_fingerprint() != next.input_fingerprint() { + return false; + } + first.input.as_ref() == next.input.as_ref() } fn can_decode_as_repeated_full_metal_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { @@ -362,43 +456,101 @@ fn is_distinct_full_color_metal_candidate(request: &QueuedRequest) -> bool { && request.backend == BackendRequest::Metal } -fn is_region_scaled_grayscale_batch_candidate(request: &QueuedRequest) -> bool { +fn is_region_scaled_direct_batch_candidate(request: &QueuedRequest) -> bool { matches!(request.op, BatchOp::RegionScaled { .. }) - && matches!(request.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) + && region_scaled_direct_format_index(request.fmt).is_some() && matches!( request.backend, BackendRequest::Auto | BackendRequest::Metal ) } -fn should_auto_use_metal_for_region_scaled_grayscale_batch(requests: &[QueuedRequest]) -> bool { - auto_region_scaled_grayscale_metal_min_dim(requests).is_some() +fn region_scaled_direct_format_index(fmt: PixelFormat) -> Option { + REGION_SCALED_DIRECT_FORMATS + .iter() + .position(|candidate| *candidate == fmt) } -fn auto_region_scaled_grayscale_metal_min_dim(requests: &[QueuedRequest]) -> Option { +fn should_auto_use_metal_for_region_scaled_direct_batch(requests: &[QueuedRequest]) -> bool { + auto_region_scaled_direct_metal_min_dim(requests).is_some() +} + +fn auto_region_scaled_direct_metal_min_dim(requests: &[QueuedRequest]) -> Option { + let first = requests.first()?; + let is_repeated_rgb = matches!( + first.fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) && can_decode_requests_as_repeated_region_scaled_batch(requests); + if matches!( + first.fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) { + if !is_repeated_rgb { + return None; + } + let repeated_rgb_eligible = requests + .iter() + .filter(|request| { + request.max_image_dim().is_some_and(|max_dim| { + max_dim >= AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM + }) + }) + .count(); + if repeated_rgb_eligible >= AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT { + return Some(AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM); + } + } + let mut count_512_class = 0usize; let mut count_1024_class = 0usize; for request in requests { let Some(max_dim) = request.max_image_dim() else { continue; }; - if max_dim >= AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_DIM { + if max_dim >= AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_DIM { count_512_class += 1; } - if max_dim >= AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_DIM { + if max_dim >= AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM { count_1024_class += 1; } } - if count_512_class >= AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_COUNT { - Some(AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_DIM) - } else if count_1024_class >= AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_COUNT { - Some(AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_DIM) + if count_512_class >= AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_COUNT { + Some(AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_DIM) + } else if count_1024_class >= AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT { + Some(AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM) } else { None } } +fn can_decode_requests_as_repeated_region_scaled_batch(requests: &[QueuedRequest]) -> bool { + let Some((first, rest)) = requests.split_first() else { + return false; + }; + !rest.is_empty() + && rest.iter().all(|request| { + is_region_scaled_direct_batch_candidate(first) + && is_region_scaled_direct_batch_candidate(request) + && first.fmt == request.fmt + && first.backend == request.backend + && same_input_bytes(first, request) + && matches!( + (first.op, request.op), + ( + BatchOp::RegionScaled { + roi: first_roi, + scale: first_scale + }, + BatchOp::RegionScaled { + roi: request_roi, + scale: request_scale + } + ) if first_roi == request_roi && first_scale == request_scale + ) + }) +} + fn can_decode_requests_as_repeated_full_grayscale_batch(requests: &[QueuedRequest]) -> bool { let Some((first, rest)) = requests.split_first() else { return false; @@ -472,7 +624,7 @@ fn process_batch(session: &mut SessionState, requests: Vec) { } if requests.len() > 1 { - if let Some(Ok(surfaces)) = decode_distinct_region_scaled_grayscale_batch(&requests) { + if let Some(Ok(surfaces)) = decode_distinct_region_scaled_direct_batch(&requests) { if surfaces.len() == requests.len() { session.submissions = session.submissions.saturating_add(1); for (request, surface) in requests.into_iter().zip(surfaces) { @@ -745,13 +897,13 @@ fn decode_distinct_full_color_batch( } } -fn decode_distinct_region_scaled_grayscale_batch( +fn decode_distinct_region_scaled_direct_batch( requests: &[QueuedRequest], ) -> Option, Error>> { let first = requests.first()?; if requests.len() <= 1 || !requests.iter().all(|request| { - is_region_scaled_grayscale_batch_candidate(request) + is_region_scaled_direct_batch_candidate(request) && request.fmt == first.fmt && request.backend == first.backend }) @@ -759,7 +911,7 @@ fn decode_distinct_region_scaled_grayscale_batch( return None; } if first.backend == BackendRequest::Auto - && !should_auto_use_metal_for_region_scaled_grayscale_batch(requests) + && !should_auto_use_metal_for_region_scaled_direct_batch(requests) { return None; } @@ -773,9 +925,22 @@ fn decode_distinct_region_scaled_grayscale_batch( _ => unreachable!("candidate op is restricted above"), }) .collect::>(); - Some( - crate::decode_region_scaled_grayscale_batch_direct_to_device(&request_specs, first.fmt), - ) + let result = match first.fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + crate::hybrid::decode_region_scaled_grayscale_batch_direct_to_device( + &request_specs, + first.fmt, + ) + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { + crate::hybrid::decode_region_scaled_color_batch_direct_to_device( + &request_specs, + first.fmt, + ) + } + _ => unreachable!("candidate pixel format is restricted above"), + }; + Some(result) } #[cfg(not(target_os = "macos"))] @@ -809,3 +974,99 @@ fn take_surface(session: &mut SessionState, slot: usize) -> Result) -> QueuedRequest { + QueuedRequest { + input, + fmt: PixelFormat::Rgb8, + backend: BackendRequest::Auto, + op: BatchOp::RegionScaled { + roi: Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }, + scale: Downscale::Quarter, + }, + output_slot: 0, + max_image_dim: OnceCell::new(), + input_fingerprint: OnceCell::new(), + } + } + + fn auto_rgb_region_scaled_request_with_max_dim( + input: Arc<[u8]>, + max_image_dim: u32, + ) -> QueuedRequest { + let request = auto_rgb_region_scaled_request(input); + request.max_image_dim.set(Some(max_image_dim)).ok(); + request + } + + #[test] + fn auto_region_scaled_rgb_threshold_requires_repeated_inputs() { + let requests = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|idx| auto_rgb_region_scaled_request(Arc::from([idx as u8]))) + .collect::>(); + + assert!(!can_decode_requests_as_repeated_region_scaled_batch( + &requests + )); + assert_eq!( + auto_region_scaled_direct_metal_min_dim(&requests), + None, + "distinct RGB ROI+scaled Auto batches must stay CPU until hybrid wins for distinct inputs" + ); + + let shared = Arc::<[u8]>::from([1_u8]); + let repeated = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|_| auto_rgb_region_scaled_request(shared.clone())) + .collect::>(); + assert!(can_decode_requests_as_repeated_region_scaled_batch( + &repeated + )); + } + + #[test] + fn auto_region_scaled_repeated_rgb_uses_measured_batch_two_metal_threshold() { + let shared = Arc::<[u8]>::from([1_u8]); + let repeated = (0..2) + .map(|_| auto_rgb_region_scaled_request_with_max_dim(shared.clone(), 512)) + .collect::>(); + + assert_eq!( + auto_region_scaled_direct_metal_min_dim(&repeated), + Some(512), + "measured repeated RGB ROI+scaled batches should route to Metal from batch 2 at 512px" + ); + + let single = vec![auto_rgb_region_scaled_request_with_max_dim(shared, 512)]; + assert_eq!(auto_region_scaled_direct_metal_min_dim(&single), None); + } + + #[test] + fn queued_request_caches_image_dimension_probe() { + let request = auto_rgb_region_scaled_request(Arc::from([0_u8])); + + assert!(!request.max_image_dim_cache_filled_for_test()); + assert_eq!(request.max_image_dim(), None); + assert!(request.max_image_dim_cache_filled_for_test()); + assert_eq!(request.max_image_dim(), None); + } + + #[test] + fn repeated_input_check_uses_pointer_identity_before_fingerprint() { + let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let first = auto_rgb_region_scaled_request(shared.clone()); + let next = auto_rgb_region_scaled_request(shared); + + assert!(same_input_bytes(&first, &next)); + assert!(!first.input_fingerprint_cache_filled_for_test()); + assert!(!next.input_fingerprint_cache_filled_for_test()); + } +} diff --git a/crates/signinum-j2k-metal/src/compute.rs b/crates/signinum-j2k-metal/src/compute.rs index 6a83d70d..e24c7d73 100644 --- a/crates/signinum-j2k-metal/src/compute.rs +++ b/crates/signinum-j2k-metal/src/compute.rs @@ -21,19 +21,22 @@ use metal::{ ComputeCommandEncoderRef, ComputePipelineState, Device, MTLCommandQueue, MTLResourceOptions, MTLSize, }; +#[cfg(target_os = "macos")] +use rayon::prelude::*; use signinum_core::{PixelFormat, Rect}; #[cfg(target_os = "macos")] use signinum_j2k_native::HtCodeBlockDecoder; use signinum_j2k_native::{ - ht_uvlc_encode_table, ht_uvlc_table0, ht_uvlc_table1, ht_vlc_encode_table0, - ht_vlc_encode_table1, ht_vlc_table0, ht_vlc_table1, ColorSpace as NativeColorSpace, - DecodedComponents as NativeDecodedComponents, EncodeProgressionOrder, EncodedHtJ2kCodeBlock, - EncodedJ2kCodeBlock, HtCodeBlockDecodeJob, HtSubBandDecodeJob, J2kCodeBlockDecodeJob, - J2kCodeBlockSegment, J2kDirectBandId, J2kDirectColorPlan, J2kDirectGrayscalePlan, + decode_ht_code_block_scalar_with_workspace, decode_j2k_code_block_scalar, ht_uvlc_encode_table, + ht_uvlc_table0, ht_uvlc_table1, ht_vlc_encode_table0, ht_vlc_encode_table1, ht_vlc_table0, + ht_vlc_table1, ColorSpace as NativeColorSpace, DecodedComponents as NativeDecodedComponents, + EncodeProgressionOrder, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, HtCodeBlockDecodeJob, + HtCodeBlockDecodeWorkspace, HtSubBandDecodeJob, J2kCodeBlockDecodeJob, J2kCodeBlockSegment, + J2kCodeBlockStyle, J2kDirectBandId, J2kDirectColorPlan, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, J2kForwardDwt53Level, J2kForwardDwt53Output, J2kHtCodeBlockEncodeJob, J2kInverseMctJob, J2kPacketizationBlockCodingMode, J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, - J2kSingleDecompositionIdwtJob, J2kStoreComponentJob, J2kSubBandDecodeJob, + J2kSingleDecompositionIdwtJob, J2kStoreComponentJob, J2kSubBandDecodeJob, J2kSubBandType, J2kTier1CodeBlockEncodeJob, J2kWaveletTransform, }; #[cfg(target_os = "macos")] @@ -52,6 +55,16 @@ use crate::{Error, Surface}; #[cfg(all(target_os = "macos", test))] static HT_BATCH_COEFFICIENT_COPY_BLITS: AtomicUsize = AtomicUsize::new(0); #[cfg(all(target_os = "macos", test))] +static HYBRID_STACKED_COMPONENT_BATCHES: AtomicUsize = AtomicUsize::new(0); +#[cfg(all(target_os = "macos", test))] +static HYBRID_CPU_DECODE_WORKER_INITS: AtomicUsize = AtomicUsize::new(0); +#[cfg(all(target_os = "macos", test))] +static HYBRID_CPU_DECODE_INPUTS: AtomicUsize = AtomicUsize::new(0); +#[cfg(all(target_os = "macos", test))] +static FLATTENED_HYBRID_CPU_DECODE_BATCHES: AtomicUsize = AtomicUsize::new(0); +#[cfg(all(target_os = "macos", test))] +static DIRECT_TIER1_INPUT_BUFFER_PREPARES: AtomicUsize = AtomicUsize::new(0); +#[cfg(all(target_os = "macos", test))] std::thread_local! { static PRIVATE_BUFFER_POOL_MISSES: Cell = const { Cell::new(0) }; static LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES: Cell = const { Cell::new(0) }; @@ -69,6 +82,56 @@ pub(crate) fn ht_batch_coefficient_copy_blits_for_test() -> usize { HT_BATCH_COEFFICIENT_COPY_BLITS.load(Ordering::Relaxed) } +#[cfg(all(target_os = "macos", test))] +pub(crate) fn reset_direct_tier1_input_buffer_prepares_for_test() { + DIRECT_TIER1_INPUT_BUFFER_PREPARES.store(0, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn direct_tier1_input_buffer_prepares_for_test() -> usize { + DIRECT_TIER1_INPUT_BUFFER_PREPARES.load(Ordering::Relaxed) +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn reset_hybrid_stacked_component_batches_for_test() { + HYBRID_STACKED_COMPONENT_BATCHES.store(0, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn hybrid_stacked_component_batches_for_test() -> usize { + HYBRID_STACKED_COMPONENT_BATCHES.load(Ordering::Relaxed) +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn reset_hybrid_cpu_decode_worker_inits_for_test() { + HYBRID_CPU_DECODE_WORKER_INITS.store(0, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn hybrid_cpu_decode_worker_inits_for_test() -> usize { + HYBRID_CPU_DECODE_WORKER_INITS.load(Ordering::Relaxed) +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn reset_hybrid_cpu_decode_inputs_for_test() { + HYBRID_CPU_DECODE_INPUTS.store(0, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn hybrid_cpu_decode_inputs_for_test() -> usize { + HYBRID_CPU_DECODE_INPUTS.load(Ordering::Relaxed) +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn reset_flattened_hybrid_cpu_decode_batches_for_test() { + FLATTENED_HYBRID_CPU_DECODE_BATCHES.store(0, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", test))] +pub(crate) fn flattened_hybrid_cpu_decode_batches_for_test() -> usize { + FLATTENED_HYBRID_CPU_DECODE_BATCHES.load(Ordering::Relaxed) +} + #[cfg(all(target_os = "macos", test))] pub(crate) fn reset_private_buffer_pool_misses_for_test() { PRIVATE_BUFFER_POOL_MISSES.with(|misses| misses.set(0)); @@ -1055,6 +1118,7 @@ struct J2kStoreParams { struct J2kRepeatedStoreParams { input_width: u32, input_height: u32, + input_instance_stride: u32, source_x: u32, source_y: u32, copy_width: u32, @@ -1591,6 +1655,7 @@ struct MetalRuntime { ht_vlc_encode_table0: Buffer, ht_vlc_encode_table1: Buffer, ht_uvlc_encode_table: Buffer, + tier1_dummy_buffer: Buffer, private_buffer_pool: Mutex>>, } @@ -1829,6 +1894,7 @@ impl MetalRuntime { size_of_val(ht_uvlc_encode_table()) as u64, MTLResourceOptions::StorageModeShared, ), + tier1_dummy_buffer: device.new_buffer(1, MTLResourceOptions::StorageModeShared), private_buffer_pool: Mutex::new(HashMap::new()), }) } @@ -2238,11 +2304,63 @@ struct DirectScratchBuffer { buffer: Buffer, } +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, PartialEq, Eq)] +enum DirectTier1Mode { + Metal, + CpuUpload, +} + +#[cfg(all(target_os = "macos", test))] +fn record_direct_tier1_input_buffer_prepare() { + DIRECT_TIER1_INPUT_BUFFER_PREPARES.fetch_add(1, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", not(test)))] +fn record_direct_tier1_input_buffer_prepare() {} + +#[cfg(target_os = "macos")] +fn prepare_direct_tier1_input_buffer( + runtime: &MetalRuntime, + data: &[T], + mode: DirectTier1Mode, +) -> Buffer { + match mode { + DirectTier1Mode::Metal => { + record_direct_tier1_input_buffer_prepare(); + borrow_slice_buffer(&runtime.device, data) + } + DirectTier1Mode::CpuUpload => runtime.tier1_dummy_buffer.clone(), + } +} + +#[cfg(target_os = "macos")] +#[derive(Default)] +struct DirectHybridStageTimings { + cpu_tier1: u128, + coefficient_upload: u128, + metal_idwt_encode: u128, + metal_store_encode: u128, + metal_mct_pack_encode: u128, + command_wait: u128, + gpu_command: u128, +} + +#[cfg(target_os = "macos")] +const HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK: usize = 2; +#[cfg(target_os = "macos")] +const HYBRID_FLAT_CPU_TIER1_MIN_DIM: u32 = 1024; +#[cfg(target_os = "macos")] +const HYBRID_FLAT_CPU_TIER1_MIN_COUNT: usize = 16; +#[cfg(target_os = "macos")] +const HYBRID_FLAT_CPU_TIER1_ENV: &str = "SIGNINUM_J2K_HYBRID_FLAT_CPU_TIER1"; + #[cfg(target_os = "macos")] #[derive(Clone)] pub(crate) struct PreparedDirectGrayscalePlan { dimensions: (u32, u32), bit_depth: u8, + tier1_prepare_mode: DirectTier1Mode, steps: Vec, classic_groups: Vec, ht_groups: Vec, @@ -2320,9 +2438,16 @@ struct PreparedHtSubBand { width: u32, height: u32, coded_data: Vec, - coded_buffer: Buffer, + coded_buffer: Option, jobs: Vec, - jobs_buffer: Buffer, + jobs_buffer: Option, +} + +#[cfg(target_os = "macos")] +#[derive(Clone)] +struct HtCodedArena { + data: Vec, + buffer: Buffer, } #[cfg(target_os = "macos")] @@ -2331,8 +2456,7 @@ struct PreparedHtSubBandGroup { start_step: usize, end_step: usize, total_coefficients: usize, - coded_data: Vec, - coded_buffer: Buffer, + coded_arena: HtCodedArena, jobs: Vec, jobs_buffer: Buffer, members: Vec, @@ -2715,6 +2839,7 @@ fn mct_transform_code(transform: J2kWaveletTransform) -> u32 { #[cfg(target_os = "macos")] fn prepare_classic_sub_band( job: &signinum_j2k_native::J2kOwnedSubBandPlan, + tier1_prepare_mode: DirectTier1Mode, ) -> Result { let mut jobs = Vec::with_capacity(job.jobs.len()); let mut coded_data = Vec::new(); @@ -2777,9 +2902,11 @@ fn prepare_classic_sub_band( } with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); - let segments_buffer = borrow_slice_buffer(&runtime.device, &segments); + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode); + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode); + let segments_buffer = + prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode); Ok(PreparedClassicSubBand { band_id: job.band_id, width: job.width, @@ -2798,6 +2925,7 @@ fn prepare_classic_sub_band( #[cfg(target_os = "macos")] fn prepare_classic_sub_band_groups( steps: &[PreparedDirectGrayscaleStep], + tier1_prepare_mode: DirectTier1Mode, ) -> Result, Error> { let mut groups = Vec::new(); let mut step_idx = 0; @@ -2811,7 +2939,10 @@ fn prepare_classic_sub_band_groups( } if sub_bands.len() > 1 { groups.push(prepare_classic_sub_band_group( - start_step, step_idx, &sub_bands, + start_step, + step_idx, + &sub_bands, + tier1_prepare_mode, )?); } if step_idx == start_step { @@ -2826,6 +2957,7 @@ fn prepare_classic_sub_band_group( start_step: usize, end_step: usize, sub_bands: &[&PreparedClassicSubBand], + tier1_prepare_mode: DirectTier1Mode, ) -> Result { let mut members = Vec::with_capacity(sub_bands.len()); let mut jobs = Vec::new(); @@ -2904,9 +3036,11 @@ fn prepare_classic_sub_band_group( } with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); - let segments_buffer = borrow_slice_buffer(&runtime.device, &segments); + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode); + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode); + let segments_buffer = + prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode); Ok(PreparedClassicSubBandGroup { start_step, end_step, @@ -2926,6 +3060,7 @@ fn prepare_classic_sub_band_group( #[cfg(target_os = "macos")] fn prepare_ht_sub_band( job: &signinum_j2k_native::HtOwnedSubBandPlan, + _tier1_prepare_mode: DirectTier1Mode, ) -> Result { let mut jobs = Vec::with_capacity(job.jobs.len()); let mut coded_data = Vec::new(); @@ -2959,24 +3094,21 @@ fn prepare_ht_sub_band( }); } - with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); - Ok(PreparedHtSubBand { - band_id: job.band_id, - width: job.width, - height: job.height, - coded_data, - coded_buffer, - jobs, - jobs_buffer, - }) + Ok(PreparedHtSubBand { + band_id: job.band_id, + width: job.width, + height: job.height, + coded_data, + coded_buffer: None, + jobs, + jobs_buffer: None, }) } #[cfg(target_os = "macos")] fn prepare_ht_sub_band_groups( steps: &[PreparedDirectGrayscaleStep], + tier1_prepare_mode: DirectTier1Mode, ) -> Result, Error> { let mut groups = Vec::new(); let mut step_idx = 0; @@ -2988,7 +3120,12 @@ fn prepare_ht_sub_band_groups( step_idx += 1; } if sub_bands.len() > 1 { - groups.push(prepare_ht_sub_band_group(start_step, step_idx, &sub_bands)?); + groups.push(prepare_ht_sub_band_group( + start_step, + step_idx, + &sub_bands, + tier1_prepare_mode, + )?); } if step_idx == start_step { step_idx += 1; @@ -3002,6 +3139,7 @@ fn prepare_ht_sub_band_group( start_step: usize, end_step: usize, sub_bands: &[&PreparedHtSubBand], + tier1_prepare_mode: DirectTier1Mode, ) -> Result { let mut members = Vec::with_capacity(sub_bands.len()); let mut jobs = Vec::new(); @@ -3053,14 +3191,17 @@ fn prepare_ht_sub_band_group( } with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode); + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode); Ok(PreparedHtSubBandGroup { start_step, end_step, total_coefficients: output_base, - coded_data, - coded_buffer, + coded_arena: HtCodedArena { + data: coded_data, + buffer: coded_buffer, + }, jobs, jobs_buffer, members, @@ -3068,21 +3209,84 @@ fn prepare_ht_sub_band_group( }) } +#[cfg(target_os = "macos")] +fn prepare_ungrouped_ht_sub_band_buffers( + steps: &mut [PreparedDirectGrayscaleStep], + groups: &[PreparedHtSubBandGroup], + tier1_prepare_mode: DirectTier1Mode, +) -> Result<(), Error> { + if tier1_prepare_mode != DirectTier1Mode::Metal { + return Ok(()); + } + + for (step_idx, step) in steps.iter_mut().enumerate() { + let PreparedDirectGrayscaleStep::HtSubBand(sub_band) = step else { + continue; + }; + if groups + .iter() + .any(|group| group.start_step <= step_idx && step_idx < group.end_step) + { + sub_band.coded_buffer = None; + sub_band.jobs_buffer = None; + continue; + } + with_runtime(|runtime| { + sub_band.coded_buffer = Some(prepare_direct_tier1_input_buffer( + runtime, + &sub_band.coded_data, + tier1_prepare_mode, + )); + sub_band.jobs_buffer = Some(prepare_direct_tier1_input_buffer( + runtime, + &sub_band.jobs, + tier1_prepare_mode, + )); + Ok(()) + })?; + } + + Ok(()) +} + +#[cfg(target_os = "macos")] +fn prepared_ht_buffer<'a>(buffer: Option<&'a Buffer>, label: &str) -> Result<&'a Buffer, Error> { + buffer.ok_or_else(|| Error::MetalKernel { + message: format!("HTJ2K MetalDirect ungrouped sub-band is missing prepared {label} buffer"), + }) +} + #[cfg(target_os = "macos")] pub(crate) fn prepare_direct_grayscale_plan( plan: &J2kDirectGrayscalePlan, +) -> Result { + prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::Metal) +} + +#[cfg(target_os = "macos")] +fn prepare_direct_grayscale_plan_for_cpu_upload( + plan: &J2kDirectGrayscalePlan, +) -> Result { + prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::CpuUpload) +} + +#[cfg(target_os = "macos")] +fn prepare_direct_grayscale_plan_with_tier1_mode( + plan: &J2kDirectGrayscalePlan, + tier1_prepare_mode: DirectTier1Mode, ) -> Result { let mut steps = Vec::with_capacity(plan.steps.len()); for step in &plan.steps { match step { J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { steps.push(PreparedDirectGrayscaleStep::ClassicSubBand( - prepare_classic_sub_band(sub_band)?, + prepare_classic_sub_band(sub_band, tier1_prepare_mode)?, )); } J2kDirectGrayscaleStep::HtSubBand(sub_band) => { steps.push(PreparedDirectGrayscaleStep::HtSubBand(prepare_ht_sub_band( sub_band, + tier1_prepare_mode, )?)); } J2kDirectGrayscaleStep::Idwt(idwt) => { @@ -3096,11 +3300,13 @@ pub(crate) fn prepare_direct_grayscale_plan( } } } - let classic_groups = prepare_classic_sub_band_groups(&steps)?; - let ht_groups = prepare_ht_sub_band_groups(&steps)?; + let classic_groups = prepare_classic_sub_band_groups(&steps, tier1_prepare_mode)?; + let ht_groups = prepare_ht_sub_band_groups(&steps, tier1_prepare_mode)?; + prepare_ungrouped_ht_sub_band_buffers(&mut steps, &ht_groups, tier1_prepare_mode)?; Ok(PreparedDirectGrayscalePlan { dimensions: plan.dimensions, bit_depth: plan.bit_depth, + tier1_prepare_mode, steps, classic_groups, ht_groups, @@ -3255,10 +3461,13 @@ fn prune_prepared_direct_grayscale_plan_to_store_windows( ); if sub_band.jobs.len() != before { sub_band.zero_fill = true; - with_runtime(|runtime| { - sub_band.jobs_buffer = borrow_slice_buffer(&runtime.device, &sub_band.jobs); - Ok(()) - })?; + if plan.tier1_prepare_mode == DirectTier1Mode::Metal { + with_runtime(|runtime| { + sub_band.jobs_buffer = + borrow_slice_buffer(&runtime.device, &sub_band.jobs); + Ok(()) + })?; + } } } PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { @@ -3268,7 +3477,7 @@ fn prune_prepared_direct_grayscale_plan_to_store_windows( required.get(&sub_band.band_id).copied(), ); if sub_band.jobs.len() != before { - compact_ht_sub_band_coded_data(sub_band)?; + compact_ht_sub_band_coded_data(sub_band, plan.tier1_prepare_mode)?; } } PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => {} @@ -3276,8 +3485,13 @@ fn prune_prepared_direct_grayscale_plan_to_store_windows( } apply_prepared_direct_idwt_output_windows(plan, &idwt_output_windows)?; - plan.classic_groups = prepare_classic_sub_band_groups(&plan.steps)?; - plan.ht_groups = prepare_ht_sub_band_groups(&plan.steps)?; + plan.classic_groups = prepare_classic_sub_band_groups(&plan.steps, plan.tier1_prepare_mode)?; + plan.ht_groups = prepare_ht_sub_band_groups(&plan.steps, plan.tier1_prepare_mode)?; + prepare_ungrouped_ht_sub_band_buffers( + &mut plan.steps, + &plan.ht_groups, + plan.tier1_prepare_mode, + )?; Ok(()) } @@ -3346,10 +3560,10 @@ fn idwt_input_windows_from_slices( hh: &DirectBandSlice, ) -> PreparedIdwtInputWindows { PreparedIdwtInputWindows { - ll: ll.window, - hl: hl.window, - lh: lh.window, - hh: hh.window, + ll: BandRequiredRegion::full(ll.window.width(), ll.window.height()), + hl: BandRequiredRegion::full(hl.window.width(), hl.window.height()), + lh: BandRequiredRegion::full(lh.window.width(), lh.window.height()), + hh: BandRequiredRegion::full(hh.window.width(), hh.window.height()), } } @@ -3454,8 +3668,8 @@ fn add_required_region( #[cfg(target_os = "macos")] fn idwt_required_output_margin(transform: J2kWaveletTransform) -> u32 { match transform { - J2kWaveletTransform::Reversible53 => 4, - J2kWaveletTransform::Irreversible97 => 12, + J2kWaveletTransform::Reversible53 => 16, + J2kWaveletTransform::Irreversible97 => 40, } } @@ -3577,7 +3791,10 @@ fn retain_ht_jobs_for_required_region( } #[cfg(target_os = "macos")] -fn compact_ht_sub_band_coded_data(sub_band: &mut PreparedHtSubBand) -> Result<(), Error> { +fn compact_ht_sub_band_coded_data( + sub_band: &mut PreparedHtSubBand, + _tier1_prepare_mode: DirectTier1Mode, +) -> Result<(), Error> { let previous = std::mem::take(&mut sub_band.coded_data); let mut compacted = Vec::new(); @@ -3599,11 +3816,9 @@ fn compact_ht_sub_band_coded_data(sub_band: &mut PreparedHtSubBand) -> Result<() } sub_band.coded_data = compacted; - with_runtime(|runtime| { - sub_band.coded_buffer = borrow_slice_buffer(&runtime.device, &sub_band.coded_data); - sub_band.jobs_buffer = borrow_slice_buffer(&runtime.device, &sub_band.jobs); - Ok(()) - }) + sub_band.coded_buffer = None; + sub_band.jobs_buffer = None; + Ok(()) } #[cfg(target_os = "macos")] @@ -3676,11 +3891,29 @@ fn crop_direct_store_step_to_output_region( #[cfg(target_os = "macos")] pub(crate) fn prepare_direct_color_plan( plan: &J2kDirectColorPlan, +) -> Result { + prepare_direct_color_plan_with_tier1_mode(plan, DirectTier1Mode::Metal) +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_direct_color_plan_for_cpu_upload( + plan: &J2kDirectColorPlan, +) -> Result { + prepare_direct_color_plan_with_tier1_mode(plan, DirectTier1Mode::CpuUpload) +} + +#[cfg(target_os = "macos")] +fn prepare_direct_color_plan_with_tier1_mode( + plan: &J2kDirectColorPlan, + tier1_prepare_mode: DirectTier1Mode, ) -> Result { let component_plans = plan .component_plans .iter() - .map(prepare_direct_grayscale_plan) + .map(|component| match tier1_prepare_mode { + DirectTier1Mode::Metal => prepare_direct_grayscale_plan(component), + DirectTier1Mode::CpuUpload => prepare_direct_grayscale_plan_for_cpu_upload(component), + }) .collect::, _>>()?; if component_plans.len() != 3 { return Err(Error::MetalKernel { @@ -3699,6 +3932,35 @@ pub(crate) fn prepare_direct_color_plan( }) } +#[cfg(target_os = "macos")] +pub(crate) fn crop_prepared_direct_color_plan_to_output_region( + plan: &mut PreparedDirectColorPlan, + region: Rect, +) -> Result<(), Error> { + if region.w == 0 || region.h == 0 { + return Err(Error::MetalKernel { + message: "J2K MetalDirect region-scaled color plan has an empty output region" + .to_string(), + }); + } + + for component_plan in &mut plan.component_plans { + crop_prepared_direct_grayscale_plan_to_output_region(component_plan, region)?; + if component_plan.dimensions != (region.w, region.h) { + return Err(Error::MetalKernel { + message: format!( + "J2K MetalDirect color component crop produced {:?}, expected {:?}", + component_plan.dimensions, + (region.w, region.h) + ), + }); + } + } + + plan.dimensions = (region.w, region.h); + Ok(()) +} + #[cfg(target_os = "macos")] impl PreparedDirectGrayscalePlan { fn classic_group_starting_at(&self, step_idx: usize) -> Option<&PreparedClassicSubBandGroup> { @@ -3993,114 +4255,1132 @@ fn encode_prepared_direct_grayscale_plan_in_command_buffer( } #[cfg(target_os = "macos")] -fn encode_prepared_direct_component_plane_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - plan: &PreparedDirectGrayscalePlan, - retained_buffers: &mut Vec, - status_checks: &mut Vec, - scratch_buffers: &mut Vec, -) -> Result { - let encoder = command_buffer.new_compute_command_encoder(); - let result = (|| { - let mut bands = Vec::::new(); - let mut final_plane = None; - let mut step_idx = 0; +fn decode_prepared_classic_sub_band_on_cpu( + sub_band: &PreparedClassicSubBand, +) -> Result, Error> { + let len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "classic J2K MetalDirect hybrid sub-band size overflow", + )?; + let mut output = vec![0.0_f32; len]; + decode_prepared_classic_jobs_on_cpu( + &sub_band.coded_data, + &sub_band.segments, + &sub_band.jobs, + &mut output, + )?; + Ok(output) +} - while step_idx < plan.steps.len() { - if let Some(group) = plan.classic_group_starting_at(step_idx) { - let output = take_f32_scratch_buffer(runtime, group.total_coefficients); - let (buffers, status_check) = - encode_prepared_classic_sub_band_group_to_buffer_in_encoder( - runtime, - encoder, - group, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } +#[cfg(target_os = "macos")] +fn decode_prepared_classic_sub_band_group_on_cpu( + group: &PreparedClassicSubBandGroup, +) -> Result, Error> { + let mut output = vec![0.0_f32; group.total_coefficients]; + decode_prepared_classic_jobs_on_cpu( + &group.coded_data, + &group.segments, + &group.jobs, + &mut output, + )?; + Ok(output) +} - if let Some(group) = plan.ht_group_starting_at(step_idx) { - let output = take_f32_scratch_buffer(runtime, group.total_coefficients); - let (buffers, status_check) = - encode_prepared_ht_sub_band_group_to_buffer_in_encoder( - runtime, - encoder, - group, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } +#[cfg(target_os = "macos")] +#[derive(Default)] +struct ClassicCpuDecodeScratch { + segments: Vec, +} - match &plan.steps[step_idx] { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - let output = take_f32_scratch_buffer( - runtime, - sub_band.width as usize * sub_band.height as usize, - ); - let (buffers, status_check) = - encode_prepared_classic_sub_band_to_buffer_in_encoder( - runtime, - encoder, - sub_band, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - let output = take_f32_scratch_buffer( - runtime, - sub_band.width as usize * sub_band.height as usize, - ); - let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( - runtime, - encoder, - sub_band, - &output.buffer, +#[cfg(target_os = "macos")] +fn decode_prepared_classic_jobs_on_cpu( + coded_data: &[u8], + segments: &[J2kClassicSegment], + jobs: &[J2kClassicCleanupBatchJob], + output: &mut [f32], +) -> Result<(), Error> { + let mut scratch = ClassicCpuDecodeScratch::default(); + decode_prepared_classic_jobs_on_cpu_with_scratch( + coded_data, + segments, + jobs, + output, + &mut scratch, + ) +} + +#[cfg(target_os = "macos")] +fn decode_prepared_classic_jobs_on_cpu_with_scratch( + coded_data: &[u8], + segments: &[J2kClassicSegment], + jobs: &[J2kClassicCleanupBatchJob], + output: &mut [f32], + scratch: &mut ClassicCpuDecodeScratch, +) -> Result<(), Error> { + for job in jobs { + let start = job.output_offset as usize; + let segment_window = prepared_classic_segment_window(segments, job)?; + scratch.segments.clear(); + scratch.segments.reserve(segment_window.len()); + for segment in segment_window { + scratch.segments.push(prepared_classic_segment(segment)?); + } + let decode_job = prepared_classic_decode_job(coded_data, &scratch.segments, job)?; + let required_len = required_classic_output_len(decode_job)?; + let end = start + .checked_add(required_len) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect hybrid output offset overflow".to_string(), + })?; + let Some(output_window) = output.get_mut(start..end) else { + return Err(Error::MetalKernel { + message: "classic J2K MetalDirect hybrid output slice is too small".to_string(), + }); + }; + decode_j2k_code_block_scalar(decode_job, output_window).map_err(native_decode_error)?; + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn prepared_classic_segment_window<'a>( + segments: &'a [J2kClassicSegment], + job: &J2kClassicCleanupBatchJob, +) -> Result<&'a [J2kClassicSegment], Error> { + let segment_start = job.segment_offset as usize; + let segment_end = segment_start + .checked_add(job.segment_count as usize) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect hybrid segment span overflow".to_string(), + })?; + segments + .get(segment_start..segment_end) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect hybrid segment span is invalid".to_string(), + }) +} + +#[cfg(target_os = "macos")] +fn prepared_classic_decode_job<'a>( + coded_data: &'a [u8], + segments: &'a [J2kCodeBlockSegment], + job: &J2kClassicCleanupBatchJob, +) -> Result, Error> { + Ok(J2kCodeBlockDecodeJob { + data: coded_data, + segments, + width: job.width, + height: job.height, + output_stride: job.output_stride as usize, + missing_bit_planes: checked_u8(job.missing_msbs, "classic missing bit planes")?, + number_of_coding_passes: checked_u8(job.number_of_coding_passes, "classic coding passes")?, + total_bitplanes: checked_u8(job.total_bitplanes, "classic total bitplanes")?, + sub_band_type: prepared_classic_sub_band_type(job.sub_band_type)?, + style: prepared_classic_style(job.style_flags), + strict: job.strict != 0, + dequantization_step: job.dequantization_step, + }) +} + +#[cfg(target_os = "macos")] +fn prepared_classic_segment(segment: &J2kClassicSegment) -> Result { + Ok(J2kCodeBlockSegment { + data_offset: segment.data_offset, + data_length: segment.data_length, + start_coding_pass: checked_u8(segment.start_coding_pass, "classic segment start pass")?, + end_coding_pass: checked_u8(segment.end_coding_pass, "classic segment end pass")?, + use_arithmetic: segment.use_arithmetic != 0, + }) +} + +#[cfg(target_os = "macos")] +fn prepared_classic_sub_band_type(value: u32) -> Result { + match value { + 0 => Ok(J2kSubBandType::LowLow), + 1 => Ok(J2kSubBandType::HighLow), + 2 => Ok(J2kSubBandType::LowHigh), + 3 => Ok(J2kSubBandType::HighHigh), + _ => Err(Error::MetalKernel { + message: format!("classic J2K MetalDirect hybrid sub-band type {value} is invalid"), + }), + } +} + +#[cfg(target_os = "macos")] +fn prepared_classic_style(flags: u32) -> J2kCodeBlockStyle { + J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: (flags + & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) + != 0, + reset_context_probabilities: (flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0, + termination_on_each_pass: (flags & J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS) != 0, + vertically_causal_context: (flags & J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT) != 0, + segmentation_symbols: (flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0, + } +} + +#[cfg(target_os = "macos")] +fn decode_prepared_ht_sub_band_on_cpu(sub_band: &PreparedHtSubBand) -> Result, Error> { + let len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "HTJ2K MetalDirect hybrid sub-band size overflow", + )?; + let mut output = vec![0.0_f32; len]; + decode_prepared_ht_jobs_on_cpu(&sub_band.coded_data, &sub_band.jobs, &mut output)?; + Ok(output) +} + +#[cfg(target_os = "macos")] +fn decode_prepared_ht_sub_band_group_on_cpu( + group: &PreparedHtSubBandGroup, +) -> Result, Error> { + let mut output = vec![0.0_f32; group.total_coefficients]; + decode_prepared_ht_jobs_on_cpu(&group.coded_arena.data, &group.jobs, &mut output)?; + Ok(output) +} + +#[cfg(target_os = "macos")] +fn decode_prepared_ht_jobs_on_cpu( + coded_data: &[u8], + jobs: &[J2kHtCleanupBatchJob], + output: &mut [f32], +) -> Result<(), Error> { + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + decode_prepared_ht_jobs_on_cpu_with_workspace(coded_data, jobs, output, &mut workspace) +} + +#[cfg(target_os = "macos")] +fn decode_prepared_ht_jobs_on_cpu_with_workspace( + coded_data: &[u8], + jobs: &[J2kHtCleanupBatchJob], + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, +) -> Result<(), Error> { + for job in jobs { + let start = job.output_offset as usize; + let decode_job = prepared_ht_decode_job(coded_data, job)?; + let required_len = required_ht_output_len(decode_job)?; + let end = start + .checked_add(required_len) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid output offset overflow".to_string(), + })?; + let Some(output_window) = output.get_mut(start..end) else { + return Err(Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid output slice is too small".to_string(), + }); + }; + decode_ht_code_block_scalar_with_workspace(decode_job, output_window, workspace) + .map_err(native_decode_error)?; + } + Ok(()) +} + +#[cfg(target_os = "macos")] +struct ClassicCpuDecodeInput<'a> { + coded_data: &'a [u8], + segments: &'a [J2kClassicSegment], + jobs: &'a [J2kClassicCleanupBatchJob], + output_len: usize, +} + +#[cfg(target_os = "macos")] +struct HtCpuDecodeInput<'a> { + coded_data: &'a [u8], + jobs: &'a [J2kHtCleanupBatchJob], + output_len: usize, +} + +#[cfg(target_os = "macos")] +fn decode_classic_inputs_on_cpu_parallel( + inputs: &[ClassicCpuDecodeInput<'_>], +) -> Result, Error> { + record_hybrid_cpu_decode_inputs(inputs.len()); + let Some(output_len) = packed_cpu_decode_output_len( + inputs.iter().map(|input| input.output_len), + "classic J2K MetalDirect hybrid batch", + )? + else { + return Ok(Vec::new()); + }; + let mut coefficients = packed_cpu_decode_coefficients(inputs.len(), output_len)?; + coefficients + .par_chunks_mut(output_len) + .zip(inputs.par_iter()) + .with_min_len(HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK) + .try_for_each_init( + || { + record_hybrid_cpu_decode_worker_init(); + ClassicCpuDecodeScratch::default() + }, + |scratch, (output, input)| { + decode_prepared_classic_jobs_on_cpu_with_scratch( + input.coded_data, + input.segments, + input.jobs, + output, + scratch, + ) + }, + )?; + Ok(coefficients) +} + +#[cfg(target_os = "macos")] +fn decode_ht_inputs_on_cpu_parallel(inputs: &[HtCpuDecodeInput<'_>]) -> Result, Error> { + record_hybrid_cpu_decode_inputs(inputs.len()); + let Some(output_len) = packed_cpu_decode_output_len( + inputs.iter().map(|input| input.output_len), + "HTJ2K MetalDirect hybrid batch", + )? + else { + return Ok(Vec::new()); + }; + let mut coefficients = packed_cpu_decode_coefficients(inputs.len(), output_len)?; + coefficients + .par_chunks_mut(output_len) + .zip(inputs.par_iter()) + .with_min_len(HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK) + .try_for_each_init( + || { + record_hybrid_cpu_decode_worker_init(); + HtCodeBlockDecodeWorkspace::default() + }, + |workspace, (output, input)| { + decode_prepared_ht_jobs_on_cpu_with_workspace( + input.coded_data, + input.jobs, + output, + workspace, + ) + }, + )?; + Ok(coefficients) +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct FlattenedCpuTier1Key { + component_idx: usize, + step_idx: usize, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +enum FlattenedCpuTier1Source<'a> { + Classic { + coded_data: &'a [u8], + segments: &'a [J2kClassicSegment], + jobs: &'a [J2kClassicCleanupBatchJob], + }, + Ht { + coded_data: &'a [u8], + jobs: &'a [J2kHtCleanupBatchJob], + }, +} + +#[cfg(target_os = "macos")] +struct FlattenedCpuTier1BucketSpec<'a> { + key: FlattenedCpuTier1Key, + output_len: usize, + inputs: Vec>, +} + +#[cfg(target_os = "macos")] +#[derive(Clone)] +struct FlattenedCpuTier1Bucket { + buffer: Buffer, + output_len: usize, + input_count: usize, +} + +#[cfg(target_os = "macos")] +struct FlattenedCpuTier1Cache { + buckets: HashMap, +} + +#[cfg(target_os = "macos")] +impl FlattenedCpuTier1Cache { + fn buffer_for( + &self, + component_idx: usize, + step_idx: usize, + output_len: usize, + input_count: usize, + ) -> Result { + let key = FlattenedCpuTier1Key { + component_idx, + step_idx, + }; + let bucket = self.buckets.get(&key).ok_or_else(|| Error::MetalKernel { + message: format!( + "J2K MetalDirect flattened hybrid cache is missing component {component_idx} step {step_idx}" + ), + })?; + if bucket.output_len != output_len || bucket.input_count != input_count { + return Err(Error::MetalKernel { + message: format!( + "J2K MetalDirect flattened hybrid cache shape mismatch at component {component_idx} step {step_idx}" + ), + }); + } + Ok(bucket.buffer.clone()) + } +} + +#[cfg(target_os = "macos")] +struct FlattenedCpuTier1WorkItem<'a> { + output_len: usize, + output: FlattenedCpuTier1Output, + source: FlattenedCpuTier1Source<'a>, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +struct FlattenedCpuTier1Output(*mut f32); + +// SAFETY: Work items are constructed from non-overlapping ranges in preallocated +// packed coefficient slabs. Each pointer is written exactly once before the +// owning Vec is moved or exposed again. +#[cfg(target_os = "macos")] +unsafe impl Send for FlattenedCpuTier1Output {} + +#[cfg(target_os = "macos")] +unsafe impl Sync for FlattenedCpuTier1Output {} + +#[cfg(target_os = "macos")] +impl FlattenedCpuTier1Output { + unsafe fn as_slice_mut<'a>(self, len: usize) -> &'a mut [f32] { + unsafe { std::slice::from_raw_parts_mut(self.0, len) } + } +} + +#[cfg(target_os = "macos")] +#[derive(Default)] +struct FlattenedCpuTier1DecodeScratch { + classic: ClassicCpuDecodeScratch, +} + +#[cfg(target_os = "macos")] +fn build_flattened_cpu_tier1_cache( + runtime: &MetalRuntime, + plans: &[Arc], + stage_timings: &mut DirectHybridStageTimings, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, +) -> Result { + let specs = collect_flattened_cpu_tier1_bucket_specs(plans)?; + let decode_started = metal_profile_stages_enabled().then(Instant::now); + let decoded_buckets = decode_flattened_cpu_tier1_buckets(&specs)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + + let upload_started = metal_profile_stages_enabled().then(Instant::now); + let mut buckets = HashMap::with_capacity(specs.len()); + for (spec, coefficients) in specs.iter().zip(decoded_buckets) { + let input_count = spec.inputs.len(); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + buckets.insert( + spec.key, + FlattenedCpuTier1Bucket { + buffer, + output_len: spec.output_len, + input_count, + }, + ); + } + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + + Ok(FlattenedCpuTier1Cache { buckets }) +} + +#[cfg(target_os = "macos")] +fn collect_flattened_cpu_tier1_bucket_specs( + plans: &[Arc], +) -> Result>, Error> { + let Some(first) = plans.first() else { + return Ok(Vec::new()); + }; + let mut specs = Vec::new(); + + for component_idx in 0..3 { + let component_plans = plans + .iter() + .map(|plan| &plan.component_plans[component_idx]) + .collect::>(); + let Some(first_component) = component_plans.first().copied() else { + continue; + }; + let broadcast_tier1_inputs = component_plans + .iter() + .all(|plan| std::ptr::eq(*plan, first_component)); + let mut step_idx = 0; + while step_idx < first.component_plans[component_idx].steps.len() { + if let Some(group) = first_component.classic_group_starting_at(step_idx) { + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| { + let group = plan.classic_group_starting_at(step_idx).ok_or_else(|| { + Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid missing classic group" + .to_string(), + } + })?; + Ok(FlattenedCpuTier1Source::Classic { + coded_data: &group.coded_data, + segments: &group.segments, + jobs: &group.jobs, + }) + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len: group.total_coefficients, + inputs, + }); + step_idx = group.end_step; + continue; + } + + if let Some(group) = first_component.ht_group_starting_at(step_idx) { + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| { + let group = plan.ht_group_starting_at(step_idx).ok_or_else(|| { + Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid missing HT group" + .to_string(), + } + })?; + Ok(FlattenedCpuTier1Source::Ht { + coded_data: &group.coded_arena.data, + jobs: &group.jobs, + }) + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len: group.total_coefficients, + inputs, + }); + step_idx = group.end_step; + continue; + } + + match &first_component.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let output_len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "classic J2K MetalDirect flattened hybrid sub-band size overflow", )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(other) => { + Ok(FlattenedCpuTier1Source::Classic { + coded_data: &other.coded_data, + segments: &other.segments, + jobs: &other.jobs, + }) + } + _ => Err(Error::MetalKernel { + message: + "J2K MetalDirect flattened hybrid missing classic sub-band" + .to_string(), + }), + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len, + inputs, + }); + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let output_len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "HTJ2K MetalDirect flattened hybrid sub-band size overflow", + )?; + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::HtSubBand(other) => { + Ok(FlattenedCpuTier1Source::Ht { + coded_data: &other.coded_data, + jobs: &other.jobs, + }) + } + _ => Err(Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid missing HT sub-band" + .to_string(), + }), + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len, + inputs, + }); + } + PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => {} + } + step_idx += 1; + } + } + + Ok(specs) +} + +#[cfg(target_os = "macos")] +fn decode_flattened_cpu_tier1_buckets( + specs: &[FlattenedCpuTier1BucketSpec<'_>], +) -> Result>, Error> { + let mut buckets = specs + .iter() + .map(|spec| packed_cpu_decode_coefficients(spec.inputs.len(), spec.output_len)) + .collect::, Error>>()?; + let mut work_items = Vec::new(); + for (bucket_idx, spec) in specs.iter().enumerate() { + for (input_idx, source) in spec.inputs.iter().copied().enumerate() { + let start = + input_idx + .checked_mul(spec.output_len) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid bucket offset overflow" + .to_string(), + })?; + let end = start + .checked_add(spec.output_len) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid bucket end overflow".to_string(), + })?; + if end > buckets[bucket_idx].len() { + return Err(Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid bucket slice is too small" + .to_string(), + }); + } + let output = + FlattenedCpuTier1Output(unsafe { buckets[bucket_idx].as_mut_ptr().add(start) }); + work_items.push(FlattenedCpuTier1WorkItem { + output_len: spec.output_len, + output, + source, + }); + } + } + + record_flattened_hybrid_cpu_decode_batch(); + record_hybrid_cpu_decode_inputs(work_items.len()); + + decode_flattened_cpu_tier1_work_items_chunked(&work_items)?; + + Ok(buckets) +} + +#[cfg(target_os = "macos")] +fn decode_flattened_cpu_tier1_work_items_chunked( + work_items: &[FlattenedCpuTier1WorkItem<'_>], +) -> Result<(), Error> { + if work_items.is_empty() { + return Ok(()); + } + + let worker_count = hybrid_cpu_decode_worker_count(work_items.len()); + let chunk_size = work_items.len().div_ceil(worker_count); + std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(worker_count); + for chunk in work_items.chunks(chunk_size) { + handles.push(scope.spawn(move || { + record_hybrid_cpu_decode_worker_init(); + let mut scratch = FlattenedCpuTier1DecodeScratch::default(); + for item in chunk { + decode_flattened_cpu_tier1_work_item(item, &mut scratch)?; + } + Ok(()) + })); + } + + for handle in handles { + match handle.join() { + Ok(Ok(())) => {} + Ok(Err(error)) => return Err(error), + Err(payload) => std::panic::resume_unwind(payload), + } + } + Ok(()) + }) +} + +#[cfg(target_os = "macos")] +fn hybrid_cpu_decode_worker_count(item_count: usize) -> usize { + let available = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); + let useful = item_count + .div_ceil(HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK.max(1)) + .max(1); + available.min(useful).max(1) +} + +#[cfg(target_os = "macos")] +fn decode_flattened_cpu_tier1_work_item( + item: &FlattenedCpuTier1WorkItem<'_>, + scratch: &mut FlattenedCpuTier1DecodeScratch, +) -> Result<(), Error> { + let output = unsafe { item.output.as_slice_mut(item.output_len) }; + match item.source { + FlattenedCpuTier1Source::Classic { + coded_data, + segments, + jobs, + } => decode_prepared_classic_jobs_on_cpu_with_scratch( + coded_data, + segments, + jobs, + output, + &mut scratch.classic, + ), + FlattenedCpuTier1Source::Ht { coded_data, jobs } => { + decode_prepared_ht_jobs_on_cpu(coded_data, jobs, output) + } + } +} + +#[cfg(target_os = "macos")] +fn packed_cpu_decode_output_len( + output_lens: impl IntoIterator, + label: &str, +) -> Result, Error> { + let mut output_lens = output_lens.into_iter(); + let Some(output_len) = output_lens.next() else { + return Ok(None); + }; + if output_len == 0 { + return Ok(None); + } + if output_lens.any(|other| other != output_len) { + return Err(Error::MetalKernel { + message: format!("{label} has mixed coefficient lengths"), + }); + } + Ok(Some(output_len)) +} + +#[cfg(target_os = "macos")] +fn packed_cpu_decode_coefficients(count: usize, output_len: usize) -> Result, Error> { + let total_len = count + .checked_mul(output_len) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect hybrid packed coefficient length overflows usize".to_string(), + })?; + Ok(vec![0.0_f32; total_len]) +} + +#[cfg(target_os = "macos")] +fn flattened_hybrid_cpu_tier1_enabled() -> bool { + std::env::var_os(HYBRID_FLAT_CPU_TIER1_ENV).is_some_and(|value| { + let value = value.to_string_lossy(); + !value.is_empty() && value != "0" && value != "false" + }) +} + +#[cfg(target_os = "macos")] +fn should_flatten_hybrid_cpu_tier1_color_batch(plans: &[Arc]) -> bool { + let Some(first) = plans.first() else { + return false; + }; + plans.len() >= HYBRID_FLAT_CPU_TIER1_MIN_COUNT + && first.dimensions.0.max(first.dimensions.1) >= HYBRID_FLAT_CPU_TIER1_MIN_DIM + && !plans.iter().all(|plan| Arc::ptr_eq(plan, first)) +} + +#[cfg(target_os = "macos")] +fn prepared_ht_decode_job<'a>( + coded_data: &'a [u8], + job: &J2kHtCleanupBatchJob, +) -> Result, Error> { + let start = job.coded_offset as usize; + let len = job.coded_len as usize; + let end = start.checked_add(len).ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid coded span overflow".to_string(), + })?; + let Some(data) = coded_data.get(start..end) else { + return Err(Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid coded span is invalid".to_string(), + }); + }; + + Ok(HtCodeBlockDecodeJob { + data, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride as usize, + missing_bit_planes: checked_u8(job.missing_msbs, "HTJ2K missing bit planes")?, + number_of_coding_passes: checked_u8(job.number_of_coding_passes, "HTJ2K coding passes")?, + num_bitplanes: checked_u8(job.num_bitplanes, "HTJ2K total bitplanes")?, + stripe_causal: job.stripe_causal != 0, + strict: true, + dequantization_step: job.dequantization_step, + }) +} + +#[cfg(target_os = "macos")] +fn checked_coefficient_len(width: u32, height: u32, message: &str) -> Result { + (width as usize) + .checked_mul(height as usize) + .ok_or_else(|| Error::MetalKernel { + message: message.to_string(), + }) +} + +#[cfg(target_os = "macos")] +fn checked_u8(value: u32, label: &str) -> Result { + u8::try_from(value).map_err(|_| Error::MetalKernel { + message: format!("J2K MetalDirect hybrid {label} exceeds u8"), + }) +} + +#[cfg(target_os = "macos")] +fn native_decode_error(error: signinum_j2k_native::DecodeError) -> Error { + Error::Decode(signinum_j2k::J2kError::Backend(error.to_string())) +} + +#[cfg(target_os = "macos")] +fn upload_cpu_decoded_coefficients( + runtime: &MetalRuntime, + mut coefficients: Vec, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, +) -> Buffer { + let buffer = borrow_mut_slice_buffer(&runtime.device, &mut coefficients); + retained_buffers.push(buffer.clone()); + retained_cpu_coefficients.push(coefficients); + buffer +} + +#[cfg(target_os = "macos")] +fn elapsed_us(started: Instant) -> u128 { + started.elapsed().as_micros() +} + +#[cfg(target_os = "macos")] +fn emit_direct_hybrid_stage_timings( + timings: &DirectHybridStageTimings, + fmt: PixelFormat, + batch_count: usize, +) { + if !metal_profile_stages_enabled() { + return; + } + + let fmt_s = format!("{fmt:?}"); + let batch_count_s = batch_count.to_string(); + for (stage, elapsed_us) in [ + ("cpu_tier1", timings.cpu_tier1), + ("coefficient_upload", timings.coefficient_upload), + ("metal_idwt_encode", timings.metal_idwt_encode), + ("metal_store_encode", timings.metal_store_encode), + ("metal_mct_pack_encode", timings.metal_mct_pack_encode), + ("command_wait", timings.command_wait), + ("gpu_command", timings.gpu_command), + ] { + let elapsed_us_s = elapsed_us.to_string(); + eprintln!( + "signinum_profile codec=j2k op=decode path=metal_direct_hybrid stage={stage} fmt={fmt_s} batch_count={batch_count_s} elapsed_us={elapsed_us_s}" + ); + } +} + +#[cfg(all(target_os = "macos", test))] +fn record_hybrid_stacked_component_batch(tier1_mode: DirectTier1Mode) { + if tier1_mode == DirectTier1Mode::CpuUpload { + HYBRID_STACKED_COMPONENT_BATCHES.fetch_add(1, Ordering::Relaxed); + } +} + +#[cfg(all(target_os = "macos", not(test)))] +fn record_hybrid_stacked_component_batch(_tier1_mode: DirectTier1Mode) {} + +#[cfg(all(target_os = "macos", test))] +fn record_hybrid_cpu_decode_worker_init() { + HYBRID_CPU_DECODE_WORKER_INITS.fetch_add(1, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", not(test)))] +fn record_hybrid_cpu_decode_worker_init() {} + +#[cfg(all(target_os = "macos", test))] +fn record_hybrid_cpu_decode_inputs(count: usize) { + HYBRID_CPU_DECODE_INPUTS.fetch_add(count, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", not(test)))] +fn record_hybrid_cpu_decode_inputs(_count: usize) {} + +#[cfg(all(target_os = "macos", test))] +fn record_flattened_hybrid_cpu_decode_batch() { + FLATTENED_HYBRID_CPU_DECODE_BATCHES.fetch_add(1, Ordering::Relaxed); +} + +#[cfg(all(target_os = "macos", not(test)))] +fn record_flattened_hybrid_cpu_decode_batch() {} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn encode_prepared_direct_component_plane_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + plan: &PreparedDirectGrayscalePlan, + tier1_mode: DirectTier1Mode, + stage_timings: &mut DirectHybridStageTimings, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, + status_checks: &mut Vec, + scratch_buffers: &mut Vec, +) -> Result { + let encoder = command_buffer.new_compute_command_encoder(); + let result = (|| { + let mut bands = Vec::::new(); + let mut final_plane = None; + let mut step_idx = 0; + let profile_stages = metal_profile_stages_enabled(); + + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients); + let (buffers, status_check) = + encode_prepared_classic_sub_band_group_to_buffer_in_encoder( + runtime, + encoder, + group, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_prepared_classic_sub_band_group_on_cpu(group)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: member.offset_elements * size_of::(), + window: member.window, + }); + } + step_idx = group.end_step; + continue; + } + + if let Some(group) = plan.ht_group_starting_at(step_idx) { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients); + let (buffers, status_check) = + encode_prepared_ht_sub_band_group_to_buffer_in_encoder( + runtime, + encoder, + group, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_prepared_ht_sub_band_group_on_cpu(group)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: member.offset_elements * size_of::(), + window: member.window, + }); + } + step_idx = group.end_step; + continue; + } + + match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer( + runtime, + sub_band.width as usize * sub_band.height as usize, + ); + let (buffers, status_check) = + encode_prepared_classic_sub_band_to_buffer_in_encoder( + runtime, + encoder, + sub_band, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_prepared_classic_sub_band_on_cpu(sub_band)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; bands.push(DirectBandSlice { band_id: sub_band.band_id, - buffer: output.buffer.clone(), + buffer, + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer( + runtime, + sub_band.width as usize * sub_band.height as usize, + ); + let (buffers, status_check) = + encode_prepared_ht_sub_band_to_buffer_in_encoder( + runtime, + encoder, + sub_band, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_prepared_ht_sub_band_on_cpu(sub_band)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; + bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer, offset_bytes: 0, window: BandRequiredRegion::full(sub_band.width, sub_band.height), }); - scratch_buffers.push(output); } PreparedDirectGrayscaleStep::Idwt(idwt) => { let ll = @@ -4116,6 +5396,7 @@ fn encode_prepared_direct_component_plane_in_command_buffer( idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), ); let output = take_f32_scratch_buffer(runtime, prepared_idwt_output_len(idwt)); + let encode_started = profile_stages.then(Instant::now); match idwt.step.transform { J2kWaveletTransform::Reversible53 => { dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( @@ -4154,6 +5435,9 @@ fn encode_prepared_direct_component_plane_in_command_buffer( ); } } + if let Some(started) = encode_started { + stage_timings.metal_idwt_encode += elapsed_us(started); + } bands.push(DirectBandSlice { band_id: idwt.step.output_band_id, buffer: output.buffer.clone(), @@ -4169,6 +5453,7 @@ fn encode_prepared_direct_component_plane_in_command_buffer( runtime, store.output_width as usize * store.output_height as usize, ); + let encode_started = profile_stages.then(Instant::now); dispatch_store_component_buffer_in_encoder_with_offsets( runtime, encoder, @@ -4188,6 +5473,9 @@ fn encode_prepared_direct_component_plane_in_command_buffer( addend: store.addend, }, ); + if let Some(started) = encode_started { + stage_timings.metal_store_encode += elapsed_us(started); + } final_plane = Some(output.buffer.clone()); scratch_buffers.push(output); } @@ -4288,8 +5576,10 @@ pub(crate) fn execute_prepared_direct_grayscale_plan_batch( with_runtime(|runtime| { let command_buffer = runtime.queue.new_command_buffer(); let mut retained_buffers = Vec::new(); + let mut retained_cpu_coefficients = Vec::>::new(); let mut status_checks = Vec::new(); let mut scratch_buffers = Vec::new(); + let mut stage_timings = DirectHybridStageTimings::default(); let mut surfaces = Vec::with_capacity(plans.len()); let component_plan_refs = plans.iter().map(Arc::as_ref).collect::>(); @@ -4298,7 +5588,12 @@ pub(crate) fn execute_prepared_direct_grayscale_plan_batch( runtime, command_buffer, &component_plan_refs, + 0, + None, + DirectTier1Mode::Metal, + &mut stage_timings, &mut retained_buffers, + &mut retained_cpu_coefficients, &mut status_checks, &mut scratch_buffers, )?; @@ -4337,6 +5632,7 @@ pub(crate) fn execute_prepared_direct_grayscale_plan_batch( validate_direct_status(status_check)?; } drop(retained_buffers); + drop(retained_cpu_coefficients); recycle_scratch_buffers(runtime, scratch_buffers); Ok(surfaces) }) @@ -4367,13 +5663,72 @@ pub(crate) fn execute_prepared_direct_color_plan_with_device( pub(crate) fn execute_prepared_direct_color_plan_batch( plans: &[Arc], fmt: PixelFormat, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1(plans, fmt, DirectTier1Mode::Metal) +} + +#[cfg(target_os = "macos")] +pub(crate) fn execute_hybrid_cpu_tier1_direct_color_plan( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, +) -> Result { + let plans = [Arc::new(plan.clone())]; + let mut surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt)?; + surfaces.pop().ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect hybrid color plan produced no surface".to_string(), + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn execute_hybrid_cpu_tier1_direct_color_plan_with_device( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| { + execute_hybrid_cpu_tier1_direct_color_plan(plan, fmt) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn execute_hybrid_cpu_tier1_direct_color_plan_batch( + plans: &[Arc], + fmt: PixelFormat, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1(plans, fmt, DirectTier1Mode::CpuUpload) +} + +#[cfg(target_os = "macos")] +fn execute_direct_color_plan_batch_with_tier1( + plans: &[Arc], + fmt: PixelFormat, + tier1_mode: DirectTier1Mode, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1_options(plans, fmt, tier1_mode, false) +} + +#[cfg(all(target_os = "macos", test))] +fn execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test( + plans: &[Arc], + fmt: PixelFormat, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1_options(plans, fmt, DirectTier1Mode::CpuUpload, true) +} + +#[cfg(target_os = "macos")] +fn execute_direct_color_plan_batch_with_tier1_options( + plans: &[Arc], + fmt: PixelFormat, + tier1_mode: DirectTier1Mode, + force_flattened_cpu_tier1: bool, ) -> Result, Error> { if plans.is_empty() { return Ok(Vec::new()); } - if plans - .iter() - .any(|plan| !prepared_direct_color_plan_supports_runtime(plan, fmt)) + if tier1_mode == DirectTier1Mode::Metal + && plans + .iter() + .any(|plan| !prepared_direct_color_plan_supports_runtime(plan, fmt)) { return Err(Error::MetalKernel { message: "unsupported classic kernel input in direct component plan".to_string(), @@ -4383,24 +5738,45 @@ pub(crate) fn execute_prepared_direct_color_plan_batch( with_runtime(|runtime| { let command_buffer = runtime.queue.new_command_buffer(); let mut retained_buffers = Vec::new(); + let mut retained_cpu_coefficients = Vec::>::new(); let mut status_checks = Vec::new(); let mut scratch_buffers = Vec::new(); + let mut stage_timings = DirectHybridStageTimings::default(); + let profile_hybrid_stages = + tier1_mode == DirectTier1Mode::CpuUpload && metal_profile_stages_enabled(); if fmt == PixelFormat::Rgb8 { if let Some(surfaces) = try_encode_stacked_mct_rgb8_direct_color_batch( runtime, command_buffer, plans, + tier1_mode, + force_flattened_cpu_tier1, + &mut stage_timings, &mut retained_buffers, + &mut retained_cpu_coefficients, &mut status_checks, &mut scratch_buffers, )? { command_buffer.commit(); + let wait_started = profile_hybrid_stages.then(Instant::now); command_buffer.wait_until_completed(); + if let Some(started) = wait_started { + stage_timings.command_wait += elapsed_us(started); + } + if profile_hybrid_stages { + if let Some(duration) = completed_command_buffer_gpu_duration(command_buffer) { + stage_timings.gpu_command += duration.as_micros(); + } + } for status_check in status_checks { validate_direct_status(status_check)?; } + if tier1_mode == DirectTier1Mode::CpuUpload { + emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); + } drop(retained_buffers); + drop(retained_cpu_coefficients); recycle_scratch_buffers(runtime, scratch_buffers); return Ok(surfaces); } @@ -4414,7 +5790,10 @@ pub(crate) fn execute_prepared_direct_color_plan_batch( command_buffer, plan, fmt, + tier1_mode, + &mut stage_timings, &mut retained_buffers, + &mut retained_cpu_coefficients, &mut status_checks, &mut scratch_buffers, )?; @@ -4422,11 +5801,24 @@ pub(crate) fn execute_prepared_direct_color_plan_batch( } command_buffer.commit(); + let wait_started = profile_hybrid_stages.then(Instant::now); command_buffer.wait_until_completed(); + if let Some(started) = wait_started { + stage_timings.command_wait += elapsed_us(started); + } + if profile_hybrid_stages { + if let Some(duration) = completed_command_buffer_gpu_duration(command_buffer) { + stage_timings.gpu_command += duration.as_micros(); + } + } for status_check in status_checks { validate_direct_status(status_check)?; } + if tier1_mode == DirectTier1Mode::CpuUpload { + emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); + } drop(retained_buffers); + drop(retained_cpu_coefficients); recycle_scratch_buffers(runtime, scratch_buffers); Ok(surfaces) }) @@ -4438,12 +5830,16 @@ fn signed_sample_bias(bit_depth: u8) -> f32 { } #[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] fn encode_prepared_direct_color_plan_in_command_buffer( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, plan: &PreparedDirectColorPlan, fmt: PixelFormat, + tier1_mode: DirectTier1Mode, + stage_timings: &mut DirectHybridStageTimings, retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, status_checks: &mut Vec, scratch_buffers: &mut Vec, ) -> Result { @@ -4462,25 +5858,34 @@ fn encode_prepared_direct_color_plan_in_command_buffer( runtime, command_buffer, component_plan, + tier1_mode, + stage_timings, retained_buffers, + retained_cpu_coefficients, status_checks, scratch_buffers, )?); } if plan.mct && fmt == PixelFormat::Rgb8 { - return Ok(encode_mct_rgb8_to_surface_in_command_buffer( + let encode_started = metal_profile_stages_enabled().then(Instant::now); + let surface = encode_mct_rgb8_to_surface_in_command_buffer( runtime, command_buffer, [&planes[0], &planes[1], &planes[2]], plan.dimensions, plan.bit_depths, plan.transform, - )); + ); + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); + } + return Ok(surface); } if plan.mct { let len = plan.dimensions.0 as usize * plan.dimensions.1 as usize; + let encode_started = metal_profile_stages_enabled().then(Instant::now); status_checks.push(dispatch_inverse_mct_buffers_in_command_buffer( runtime, command_buffer, @@ -4493,6 +5898,9 @@ fn encode_prepared_direct_color_plan_in_command_buffer( signed_sample_bias(plan.bit_depths[2]), ], )?); + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); + } } let stage = PlaneStage { @@ -4513,7 +5921,13 @@ fn encode_prepared_direct_color_plan_in_command_buffer( None, ], }; - encode_plane_stage_to_surface_in_command_buffer(runtime, command_buffer, &stage, fmt) + let encode_started = metal_profile_stages_enabled().then(Instant::now); + let surface = + encode_plane_stage_to_surface_in_command_buffer(runtime, command_buffer, &stage, fmt); + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); + } + surface } #[cfg(target_os = "macos")] @@ -4593,11 +6007,16 @@ struct StackedDirectComponentPlane { } #[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] fn try_encode_stacked_mct_rgb8_direct_color_batch( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, plans: &[Arc], + tier1_mode: DirectTier1Mode, + force_flattened_cpu_tier1: bool, + stage_timings: &mut DirectHybridStageTimings, retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, status_checks: &mut Vec, scratch_buffers: &mut Vec, ) -> Result>, Error> { @@ -4618,6 +6037,22 @@ fn try_encode_stacked_mct_rgb8_direct_color_batch( return Ok(None); } + let flattened_cpu_tier1_cache = if tier1_mode == DirectTier1Mode::CpuUpload + && (force_flattened_cpu_tier1 + || flattened_hybrid_cpu_tier1_enabled() + || should_flatten_hybrid_cpu_tier1_color_batch(plans)) + { + Some(build_flattened_cpu_tier1_cache( + runtime, + plans, + stage_timings, + retained_buffers, + retained_cpu_coefficients, + )?) + } else { + None + }; + let mut stacked_planes = Vec::with_capacity(3); for component_idx in 0..3 { let component_plan_refs = plans @@ -4631,7 +6066,12 @@ fn try_encode_stacked_mct_rgb8_direct_color_batch( runtime, command_buffer, &component_plan_refs, + component_idx, + flattened_cpu_tier1_cache.as_ref(), + tier1_mode, + stage_timings, retained_buffers, + retained_cpu_coefficients, status_checks, scratch_buffers, )?); @@ -4644,6 +6084,7 @@ fn try_encode_stacked_mct_rgb8_direct_color_batch( return Ok(None); } + let encode_started = metal_profile_stages_enabled().then(Instant::now); let surfaces = encode_batched_mct_rgb8_to_surfaces_in_command_buffer( runtime, command_buffer, @@ -4657,6 +6098,9 @@ fn try_encode_stacked_mct_rgb8_direct_color_batch( first.bit_depths, first.transform, )?; + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); + } Ok(Some(surfaces)) } @@ -4752,6 +6196,41 @@ fn supports_stacked_direct_component_plane_batch(plans: &[&PreparedDirectGraysca true } +#[cfg(all(target_os = "macos", test))] +fn prepared_direct_color_tier1_input_count(plan: &PreparedDirectColorPlan) -> usize { + plan.component_plans + .iter() + .map(prepared_direct_component_tier1_input_count) + .sum() +} + +#[cfg(all(target_os = "macos", test))] +fn prepared_direct_component_tier1_input_count(plan: &PreparedDirectGrayscalePlan) -> usize { + let mut count = 0; + let mut step_idx = 0; + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + count += 1; + step_idx = group.end_step; + continue; + } + if let Some(group) = plan.ht_group_starting_at(step_idx) { + count += 1; + step_idx = group.end_step; + continue; + } + if matches!( + &plan.steps[step_idx], + PreparedDirectGrayscaleStep::ClassicSubBand(_) + | PreparedDirectGrayscaleStep::HtSubBand(_) + ) { + count += 1; + } + step_idx += 1; + } + count +} + #[cfg(target_os = "macos")] fn prepared_direct_color_plan_supports_runtime( plan: &PreparedDirectColorPlan, @@ -4769,24 +6248,27 @@ fn prepared_direct_color_plan_supports_runtime( #[cfg(target_os = "macos")] fn prepared_direct_component_plan_supports_runtime(plan: &PreparedDirectGrayscalePlan) -> bool { - plan.steps.iter().all(|step| match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => sub_band - .jobs - .iter() - .all(|job| classic_prepared_job_supports_runtime(job, &sub_band.segments)), - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - sub_band.jobs.iter().all(ht_prepared_job_supports_runtime) - } - PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => true, - }) && plan.classic_groups.iter().all(|group| { - group - .jobs + plan.tier1_prepare_mode == DirectTier1Mode::Metal + && plan.steps.iter().all(|step| match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => sub_band + .jobs + .iter() + .all(|job| classic_prepared_job_supports_runtime(job, &sub_band.segments)), + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + sub_band.jobs.iter().all(ht_prepared_job_supports_runtime) + } + PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => true, + }) + && plan.classic_groups.iter().all(|group| { + group + .jobs + .iter() + .all(|job| classic_prepared_job_supports_runtime(job, &group.segments)) + }) + && plan + .ht_groups .iter() - .all(|job| classic_prepared_job_supports_runtime(job, &group.segments)) - }) && plan - .ht_groups - .iter() - .all(|group| group.jobs.iter().all(ht_prepared_job_supports_runtime)) + .all(|group| group.jobs.iter().all(ht_prepared_job_supports_runtime)) } #[cfg(target_os = "macos")] @@ -4952,11 +6434,17 @@ fn store_shapes_match(first: &J2kDirectStoreStep, other: &J2kDirectStoreStep) -> } #[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] fn encode_stacked_direct_component_plane_batch( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, plans: &[&PreparedDirectGrayscalePlan], + component_idx: usize, + flattened_cpu_tier1_cache: Option<&FlattenedCpuTier1Cache>, + tier1_mode: DirectTier1Mode, + stage_timings: &mut DirectHybridStageTimings, retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, status_checks: &mut Vec, scratch_buffers: &mut Vec, ) -> Result { @@ -4967,9 +6455,12 @@ fn encode_stacked_direct_component_plane_batch( }; let count = plans.len(); + let broadcast_tier1_inputs = tier1_mode == DirectTier1Mode::CpuUpload + && plans.iter().all(|plan| std::ptr::eq(*plan, *first)); let mut band_sets = vec![Vec::::new(); count]; let mut final_plane = None; let mut step_idx = 0; + let profile_stages = tier1_mode == DirectTier1Mode::CpuUpload && metal_profile_stages_enabled(); while step_idx < first.steps.len() { if let Some(group) = first.classic_group_starting_at(step_idx) { @@ -4980,30 +6471,86 @@ fn encode_stacked_direct_component_plane_batch( .expect("preflight validated classic group") }) .collect::>(); - let output = take_f32_scratch_buffer(runtime, group.total_coefficients * count); - let (buffers, status_check) = - encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( - runtime, - command_buffer, - &groups, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients * count); + let (buffers, status_check) = + encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( + runtime, + command_buffer, + &groups, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_groups = if broadcast_tier1_inputs { + &groups[..1] + } else { + &groups + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + group.total_coefficients, + input_groups.len(), + )? + } else { + let inputs = input_groups + .iter() + .map(|group| ClassicCpuDecodeInput { + coded_data: &group.coded_data, + segments: &group.segments, + jobs: &group.jobs, + output_len: group.total_coefficients, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_classic_inputs_on_cpu_parallel(&inputs)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; let stride_bytes = group.total_coefficients * size_of::(); for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - for member in &groups[instance_idx].members { + let source_group = if broadcast_tier1_inputs { + groups[0] + } else { + groups[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; + for member in &source_group.members { bands.push(DirectBandSlice { band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), + buffer: buffer.clone(), + offset_bytes: instance_offset + member.offset_elements * size_of::(), window: member.window, }); } } - scratch_buffers.push(output); step_idx = group.end_step; continue; } @@ -5016,29 +6563,84 @@ fn encode_stacked_direct_component_plane_batch( .expect("preflight validated HT group") }) .collect::>(); - let output = take_f32_scratch_buffer(runtime, group.total_coefficients * count); - let (buffers, status_check) = - encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( - runtime, - command_buffer, - &groups, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients * count); + let (buffers, status_check) = + encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( + runtime, + command_buffer, + &groups, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_groups = if broadcast_tier1_inputs { + &groups[..1] + } else { + &groups + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + group.total_coefficients, + input_groups.len(), + )? + } else { + let inputs = input_groups + .iter() + .map(|group| HtCpuDecodeInput { + coded_data: &group.coded_arena.data, + jobs: &group.jobs, + output_len: group.total_coefficients, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_ht_inputs_on_cpu_parallel(&inputs)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; let stride_bytes = group.total_coefficients * size_of::(); for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - for member in &groups[instance_idx].members { + let source_group = if broadcast_tier1_inputs { + groups[0] + } else { + groups[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; + for member in &source_group.members { bands.push(DirectBandSlice { band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), + buffer: buffer.clone(), + offset_bytes: instance_offset + member.offset_elements * size_of::(), window: member.window, }); } } - scratch_buffers.push(output); step_idx = group.end_step; continue; } @@ -5053,30 +6655,87 @@ fn encode_stacked_direct_component_plane_batch( }) .collect::>(); let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( - runtime, - command_buffer, - &sub_bands, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, per_instance_len * count); + let (buffers, status_check) = + encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( + runtime, + command_buffer, + &sub_bands, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_sub_bands = if broadcast_tier1_inputs { + &sub_bands[..1] + } else { + &sub_bands + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + per_instance_len, + input_sub_bands.len(), + )? + } else { + let inputs = input_sub_bands + .iter() + .map(|sub_band| ClassicCpuDecodeInput { + coded_data: &sub_band.coded_data, + segments: &sub_band.segments, + jobs: &sub_band.jobs, + output_len: per_instance_len, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_classic_inputs_on_cpu_parallel(&inputs)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; let stride_bytes = per_instance_len * size_of::(); for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + let source_sub_band = if broadcast_tier1_inputs { + sub_bands[0] + } else { + sub_bands[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; bands.push(DirectBandSlice { - band_id: sub_bands[instance_idx].band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, + band_id: source_sub_band.band_id, + buffer: buffer.clone(), + offset_bytes: instance_offset, window: BandRequiredRegion::full( - sub_bands[instance_idx].width, - sub_bands[instance_idx].height, + source_sub_band.width, + source_sub_band.height, ), }); } - scratch_buffers.push(output); } PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { let sub_bands = plans @@ -5087,33 +6746,90 @@ fn encode_stacked_direct_component_plane_batch( }) .collect::>(); let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_distinct_ht_sub_bands_to_buffer_in_command_buffer( - runtime, - command_buffer, - &sub_bands, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, per_instance_len * count); + let (buffers, status_check) = + encode_distinct_ht_sub_bands_to_buffer_in_command_buffer( + runtime, + command_buffer, + &sub_bands, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_sub_bands = if broadcast_tier1_inputs { + &sub_bands[..1] + } else { + &sub_bands + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + per_instance_len, + input_sub_bands.len(), + )? + } else { + let inputs = input_sub_bands + .iter() + .map(|sub_band| HtCpuDecodeInput { + coded_data: &sub_band.coded_data, + jobs: &sub_band.jobs, + output_len: per_instance_len, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let coefficients = decode_ht_inputs_on_cpu_parallel(&inputs)?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; let stride_bytes = per_instance_len * size_of::(); for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + let source_sub_band = if broadcast_tier1_inputs { + sub_bands[0] + } else { + sub_bands[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; bands.push(DirectBandSlice { - band_id: sub_bands[instance_idx].band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, + band_id: source_sub_band.band_id, + buffer: buffer.clone(), + offset_bytes: instance_offset, window: BandRequiredRegion::full( - sub_bands[instance_idx].width, - sub_bands[instance_idx].height, + source_sub_band.width, + source_sub_band.height, ), }); } - scratch_buffers.push(output); } PreparedDirectGrayscaleStep::Idwt(idwt) => { let per_instance_len = prepared_idwt_output_len(idwt); let output = take_f32_scratch_buffer(runtime, per_instance_len * count); + let encode_started = profile_stages.then(Instant::now); match idwt.step.transform { J2kWaveletTransform::Reversible53 => { let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( @@ -5214,6 +6930,9 @@ fn encode_stacked_direct_component_plane_batch( } } } + if let Some(started) = encode_started { + stage_timings.metal_idwt_encode += elapsed_us(started); + } let stride_bytes = per_instance_len * size_of::(); for (instance_idx, bands) in band_sets.iter_mut().enumerate() { let PreparedDirectGrayscaleStep::Idwt(step) = @@ -5231,18 +6950,24 @@ fn encode_stacked_direct_component_plane_batch( scratch_buffers.push(output); } PreparedDirectGrayscaleStep::Store(store) => { - let (input, _) = - lookup_direct_band_slice(&band_sets[0], store.input_band_id, store.input_rect)?; + let (input, input_instance_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + store.input_band_id, + store.input_rect, + )?; let per_instance_len = store.output_width as usize * store.output_height as usize; let output = take_f32_scratch_buffer(runtime, per_instance_len * count); + let encode_started = profile_stages.then(Instant::now); dispatch_store_component_repeated_in_command_buffer( runtime, command_buffer, - &input, + &input.buffer, + input.offset_bytes, &output.buffer, J2kRepeatedStoreParams { input_width: store.input_rect.width(), input_height: store.input_rect.height(), + input_instance_stride, source_x: store.source_x, source_y: store.source_y, copy_width: store.copy_width, @@ -5258,6 +6983,9 @@ fn encode_stacked_direct_component_plane_batch( })?, }, ); + if let Some(started) = encode_started { + stage_timings.metal_store_encode += elapsed_us(started); + } final_plane = Some(output.buffer.clone()); scratch_buffers.push(output); } @@ -5268,6 +6996,7 @@ fn encode_stacked_direct_component_plane_batch( let buffer = final_plane.ok_or_else(|| Error::MetalKernel { message: "J2K MetalDirect color component batch did not produce a final plane".to_string(), })?; + record_hybrid_stacked_component_batch(tier1_mode); Ok(StackedDirectComponentPlane { buffer, dimensions: first.dimensions, @@ -5595,10 +7324,19 @@ fn encode_repeated_direct_grayscale_plan_in_command_buffer( runtime, command_buffer, &input, + 0, &output.buffer, J2kRepeatedStoreParams { input_width: store.input_rect.width(), input_height: store.input_rect.height(), + input_instance_stride: store + .input_rect + .width() + .checked_mul(store.input_rect.height()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated store input stride overflows u32" + .to_string(), + })?, source_x: store.source_x, source_y: store.source_y, copy_width: store.copy_width, @@ -7964,12 +9702,13 @@ fn dispatch_store_component_repeated_in_command_buffer( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, input: &Buffer, + input_offset_bytes: usize, output: &Buffer, params: J2kRepeatedStoreParams, ) { let encoder = command_buffer.new_compute_command_encoder(); encoder.set_compute_pipeline_state(&runtime.store_component_repeated); - encoder.set_buffer(0, Some(input), 0); + encoder.set_buffer(0, Some(input), input_offset_bytes as u64); encoder.set_buffer(1, Some(output), 0); encoder.set_bytes( 2, @@ -9475,7 +11214,7 @@ fn encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( .iter() .enumerate() .map(|(index, group)| DistinctHtBatch { - coded_data: &group.coded_data, + coded_data: &group.coded_arena.data, jobs: &group.jobs, output_base: index * per_instance_len, }), @@ -9948,8 +11687,8 @@ fn encode_repeated_ht_sub_band_to_buffer_in_command_buffer( .ok_or_else(|| Error::MetalKernel { message: "HTJ2K MetalDirect repeated job count overflow".to_string(), })?; - let coded_buffer = job.coded_buffer.clone(); - let jobs_buffer = job.jobs_buffer.clone(); + let coded_buffer = prepared_ht_buffer(job.coded_buffer.as_ref(), "coded")?.clone(); + let jobs_buffer = prepared_ht_buffer(job.jobs_buffer.as_ref(), "jobs")?.clone(); let status_check = dispatch_ht_cleanup_repeated_batched_in_command_buffer( runtime, command_buffer, @@ -9991,7 +11730,7 @@ fn encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer( .ok_or_else(|| Error::MetalKernel { message: "HTJ2K MetalDirect repeated grouped job count overflow".to_string(), })?; - let coded_buffer = group.coded_buffer.clone(); + let coded_buffer = group.coded_arena.buffer.clone(); let jobs_buffer = group.jobs_buffer.clone(); let status_check = dispatch_ht_cleanup_repeated_batched_in_command_buffer( runtime, @@ -10032,8 +11771,8 @@ fn encode_prepared_ht_sub_band_to_buffer_in_encoder( )); } - let coded_buffer = job.coded_buffer.clone(); - let jobs_buffer = job.jobs_buffer.clone(); + let coded_buffer = prepared_ht_buffer(job.coded_buffer.as_ref(), "coded")?.clone(); + let jobs_buffer = prepared_ht_buffer(job.jobs_buffer.as_ref(), "jobs")?.clone(); let status_check = dispatch_ht_cleanup_batched_in_encoder( runtime, encoder, @@ -10067,7 +11806,7 @@ fn encode_prepared_ht_sub_band_group_to_buffer_in_encoder( )); } - let coded_buffer = group.coded_buffer.clone(); + let coded_buffer = group.coded_arena.buffer.clone(); let jobs_buffer = group.jobs_buffer.clone(); let status_check = dispatch_ht_cleanup_batched_in_encoder( runtime, @@ -14913,21 +16652,39 @@ pub(crate) fn decode_ht_cleanup_sub_band( mod tests { use super::{ classic_batch_uses_plain_fast_path, classic_repeated_uses_plain_fast_path, - crop_prepared_direct_grayscale_plan_to_output_region, decode_scaled_to_surface, + crop_prepared_direct_grayscale_plan_to_output_region, + decode_prepared_classic_sub_band_on_cpu, decode_scaled_to_surface, + direct_tier1_input_buffer_prepares_for_test, + execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test, + execute_hybrid_cpu_tier1_direct_color_plan_batch, + flattened_hybrid_cpu_decode_batches_for_test, hybrid_cpu_decode_inputs_for_test, + hybrid_cpu_decode_worker_inits_for_test, hybrid_stacked_component_batches_for_test, j2k_pack_kernel_name_for, j2k_pack_scale_arrays, output_shape_for, - prepare_direct_grayscale_plan, prepared_direct_grayscale_plan_compute_encoder_count, - prepared_idwt_output_len, prepared_repeated_direct_ht_cleanup_dispatch_count, - repeated_gray_store_is_contiguous_full_surface, runtime_initialization_error, - supports_stacked_direct_component_plane_batch, with_runtime_for_device, - J2kClassicCleanupBatchJob, J2kClassicSegment, J2kRepeatedGrayStoreParams, MetalRuntime, + prepare_direct_color_plan, prepare_direct_color_plan_for_cpu_upload, + prepare_direct_grayscale_plan, prepared_direct_color_tier1_input_count, + prepared_direct_grayscale_plan_compute_encoder_count, prepared_idwt_output_len, + prepared_repeated_direct_ht_cleanup_dispatch_count, + repeated_gray_store_is_contiguous_full_surface, + reset_direct_tier1_input_buffer_prepares_for_test, + reset_flattened_hybrid_cpu_decode_batches_for_test, + reset_hybrid_cpu_decode_inputs_for_test, reset_hybrid_cpu_decode_worker_inits_for_test, + reset_hybrid_stacked_component_batches_for_test, runtime_initialization_error, + should_flatten_hybrid_cpu_tier1_color_batch, supports_stacked_direct_component_plane_batch, + with_runtime_for_device, J2kClassicCleanupBatchJob, J2kClassicSegment, + J2kRepeatedGrayStoreParams, MetalRuntime, PreparedClassicSubBand, PreparedDirectColorPlan, PreparedDirectGrayscaleStep, }; use metal::Device; use signinum_core::PixelFormat; use signinum_j2k_native::{ - encode, encode_htj2k, ColorSpace as NativeColorSpace, DecodeSettings, DecoderContext, - EncodeOptions, Image, + decode_j2k_sub_band_scalar, encode, encode_htj2k, ColorSpace as NativeColorSpace, + DecodeSettings, DecoderContext, EncodeOptions, Image, J2kCodeBlockBatchJob, + J2kCodeBlockDecodeJob, J2kDirectGrayscaleStep as NativeDirectGrayscaleStep, + J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kSubBandDecodeJob, J2kWaveletTransform, }; + use std::sync::{Arc, Mutex}; + + static HYBRID_COUNTER_TEST_LOCK: Mutex<()> = Mutex::new(()); #[test] fn rgb16_with_alpha_is_rejected() { @@ -15092,6 +16849,173 @@ mod tests { )); } + #[test] + fn grouped_ht_direct_plan_uses_one_group_coded_arena() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared.ht_groups.len(), + 1, + "fixture must exercise one grouped HT dispatch" + ); + let group = &prepared.ht_groups[0]; + assert!(!group.coded_arena.data.is_empty()); + assert_eq!( + direct_tier1_input_buffer_prepares_for_test(), + 2, + "grouped HT dispatch should prepare one coded arena buffer and one job buffer" + ); + + for step in &prepared.steps[group.start_step..group.end_step] { + let PreparedDirectGrayscaleStep::HtSubBand(sub_band) = step else { + panic!("HT group should only span HT sub-band steps"); + }; + assert!(sub_band.coded_buffer.is_none()); + assert!(sub_band.jobs_buffer.is_none()); + } + } + + #[test] + fn prepared_classic_sub_band_decodes_on_cpu_for_hybrid_upload() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode classic gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let native_sub_band = first_native_classic_sub_band(&plan); + let prepared_sub_band = first_prepared_classic_sub_band(&prepared); + + let expected = decode_native_classic_sub_band(native_sub_band); + let actual = decode_prepared_classic_sub_band_on_cpu(prepared_sub_band) + .expect("prepared CPU decode"); + + assert_eq!(actual, expected); + } + + #[test] + fn cpu_upload_color_prepare_skips_tier1_metal_input_buffers() { + if Device::system_default().is_none() { + eprintln!("skipping CPUUpload prepare test: no Metal device"); + return; + } + + let pixels = signinum_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let metal_prepared = prepare_direct_color_plan(&plan).expect("Metal prepared color plan"); + assert_eq!(metal_prepared.component_plans.len(), 3); + assert!( + direct_tier1_input_buffer_prepares_for_test() > 0, + "normal Metal preparation should build Tier-1 input buffers" + ); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let cpu_upload_prepared = + prepare_direct_color_plan_for_cpu_upload(&plan).expect("CPUUpload prepared color plan"); + assert_eq!(cpu_upload_prepared.component_plans.len(), 3); + assert_eq!( + direct_tier1_input_buffer_prepares_for_test(), + 0, + "CPUUpload preparation should keep coded Tier-1 payloads on CPU and skip Metal input buffers" + ); + } + + fn first_native_classic_sub_band( + plan: &signinum_j2k_native::J2kDirectGrayscalePlan, + ) -> &J2kOwnedSubBandPlan { + plan.steps + .iter() + .find_map(|step| match step { + NativeDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), + _ => None, + }) + .expect("classic sub-band step") + } + + fn first_prepared_classic_sub_band( + plan: &super::PreparedDirectGrayscalePlan, + ) -> &PreparedClassicSubBand { + plan.steps + .iter() + .find_map(|step| match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), + _ => None, + }) + .expect("prepared classic sub-band step") + } + + fn decode_native_classic_sub_band(plan: &J2kOwnedSubBandPlan) -> Vec { + let mut output = vec![0.0_f32; plan.width as usize * plan.height as usize]; + let jobs = plan + .jobs + .iter() + .map(|job| J2kCodeBlockBatchJob { + output_x: job.output_x, + output_y: job.output_y, + code_block: native_classic_job(job), + }) + .collect::>(); + decode_j2k_sub_band_scalar( + J2kSubBandDecodeJob { + width: plan.width, + height: plan.height, + jobs: &jobs, + }, + &mut output, + ) + .expect("native scalar classic sub-band decode"); + output + } + + fn native_classic_job(job: &J2kOwnedCodeBlockBatchJob) -> J2kCodeBlockDecodeJob<'_> { + J2kCodeBlockDecodeJob { + data: &job.data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + } + } + #[test] fn prepared_ht_direct_plan_encodes_full_decode_in_one_compute_encoder() { let pixels: Vec = (0..64).collect(); @@ -15190,11 +17114,239 @@ mod tests { ); } + #[test] + fn hybrid_rgb8_batch_uses_stacked_component_graph() { + let pixels = signinum_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_stacked_component_batches_for_test(); + reset_hybrid_cpu_decode_worker_inits_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared], + PixelFormat::Rgb8, + ) + .expect("hybrid RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert!( + hybrid_stacked_component_batches_for_test() >= 3, + "hybrid RGB batch should stack each component plane instead of encoding each tile/component serially" + ); + assert!( + hybrid_cpu_decode_worker_inits_for_test() > 0, + "hybrid RGB batch should use worker-local CPU decode scratch instead of per-input decode/flatten" + ); + } + + #[test] + fn hybrid_rgb8_repeated_batch_decodes_shared_tier1_inputs_once() { + let pixels = signinum_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let unique_tier1_inputs = prepared_direct_color_tier1_input_count(&prepared); + assert!( + unique_tier1_inputs > 0, + "fixture should have Tier-1 inputs to decode" + ); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared.clone(), prepared], + PixelFormat::Rgb8, + ) + .expect("hybrid repeated RGB8 batch"); + + assert_eq!(surfaces.len(), 3); + assert_eq!( + hybrid_cpu_decode_inputs_for_test(), + unique_tier1_inputs, + "repeated RGB hybrid batches should decode each shared coefficient input once, not once per output surface" + ); + } + + #[test] + fn hybrid_rgb8_distinct_batch_keeps_tier1_inputs_separate() { + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes_a = encode( + &signinum_test_support::gradient_variant_u8(32, 32, 3, 0), + 32, + 32, + 3, + 8, + false, + &options, + ) + .expect("encode first rgb8"); + let bytes_b = encode( + &signinum_test_support::gradient_variant_u8(32, 32, 3, 7), + 32, + 32, + 3, + 8, + false, + &options, + ) + .expect("encode second rgb8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_color_plan_with_context(&mut context_a) + .expect("first direct color plan"); + let plan_b = image_b + .build_direct_color_plan_with_context(&mut context_b) + .expect("second direct color plan"); + let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); + let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); + let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) + + prepared_direct_color_tier1_input_count(&prepared_b); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared_a, prepared_b], + PixelFormat::Rgb8, + ) + .expect("hybrid distinct RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert_ne!( + surfaces[0].as_bytes(), + surfaces[1].as_bytes(), + "distinct RGB inputs must not reuse the first tile's decoded coefficients" + ); + assert_eq!( + hybrid_cpu_decode_inputs_for_test(), + expected_inputs, + "distinct RGB hybrid batches should decode each tile's own Tier-1 inputs" + ); + } + + #[test] + fn hybrid_rgb8_flattened_cpu_tier1_batch_uses_one_decode_queue() { + let pixels_a = signinum_test_support::gradient_variant_u8(32, 32, 3, 0); + let pixels_b = signinum_test_support::gradient_variant_u8(32, 32, 3, 11); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes_a = encode(&pixels_a, 32, 32, 3, 8, false, &options).expect("encode first rgb8"); + let bytes_b = encode(&pixels_b, 32, 32, 3, 8, false, &options).expect("encode second rgb8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_color_plan_with_context(&mut context_a) + .expect("first direct color plan"); + let plan_b = image_b + .build_direct_color_plan_with_context(&mut context_b) + .expect("second direct color plan"); + let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); + let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); + let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) + + prepared_direct_color_tier1_input_count(&prepared_b); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + reset_flattened_hybrid_cpu_decode_batches_for_test(); + + let surfaces = execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test( + &[prepared_a, prepared_b], + PixelFormat::Rgb8, + ) + .expect("flattened hybrid distinct RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert_ne!( + surfaces[0].as_bytes(), + surfaces[1].as_bytes(), + "flattened distinct RGB hybrid batches must keep each tile's coefficients separate" + ); + assert_eq!( + hybrid_cpu_decode_inputs_for_test(), + expected_inputs, + "flattened RGB hybrid batches should still decode every distinct Tier-1 input" + ); + assert_eq!( + flattened_hybrid_cpu_decode_batches_for_test(), + 1, + "flattened RGB hybrid should collect Tier-1 work into one CPU decode queue" + ); + } + + #[test] + fn flattened_cpu_tier1_default_gate_targets_large_distinct_batches_only() { + fn color_plan(width: u32, height: u32) -> Arc { + Arc::new(PreparedDirectColorPlan { + dimensions: (width, height), + bit_depths: [8, 8, 8], + mct: true, + transform: J2kWaveletTransform::Reversible53, + component_plans: Vec::new(), + }) + } + + let repeated = vec![color_plan(1024, 1024); 16]; + assert!( + !should_flatten_hybrid_cpu_tier1_color_batch(&repeated), + "repeated RGB batches already win through shared Tier-1 decode and should not use the flattened distinct scheduler" + ); + + let small_distinct = (0..16).map(|_| color_plan(256, 256)).collect::>(); + assert!( + !should_flatten_hybrid_cpu_tier1_color_batch(&small_distinct), + "small RGB batches measured slower with flattened Tier-1 and should stay on the grouped path" + ); + + let large_distinct = (0..16).map(|_| color_plan(1024, 1024)).collect::>(); + assert!( + should_flatten_hybrid_cpu_tier1_color_batch(&large_distinct), + "large distinct RGB explicit hybrid batches measured faster with flattened Tier-1" + ); + } + #[test] fn cropped_region_scaled_ht_direct_plan_prunes_codeblocks_outside_output_roi() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { pixels.push(((x * 3 + y * 5) & 0xff) as u8); } } @@ -15206,11 +17358,11 @@ mod tests { ..EncodeOptions::default() }; let bytes = - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); + encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); let image = Image::new( &bytes, &DecodeSettings { - target_resolution: Some((32, 32)), + target_resolution: Some((64, 64)), ..DecodeSettings::default() }, ) @@ -15229,10 +17381,10 @@ mod tests { crop_prepared_direct_grayscale_plan_to_output_region( &mut prepared, signinum_core::Rect { - x: 10, - y: 10, - w: 4, - h: 4, + x: 24, + y: 24, + w: 8, + h: 8, }, ) .expect("crop direct plan"); @@ -15246,9 +17398,9 @@ mod tests { #[test] fn cropped_region_scaled_ht_direct_plan_compacts_coded_payloads() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { pixels.push(((x * 3 + y * 5) & 0xff) as u8); } } @@ -15260,11 +17412,11 @@ mod tests { ..EncodeOptions::default() }; let bytes = - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); + encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); let image = Image::new( &bytes, &DecodeSettings { - target_resolution: Some((32, 32)), + target_resolution: Some((64, 64)), ..DecodeSettings::default() }, ) @@ -15280,10 +17432,10 @@ mod tests { crop_prepared_direct_grayscale_plan_to_output_region( &mut prepared, signinum_core::Rect { - x: 10, - y: 10, - w: 4, - h: 4, + x: 24, + y: 24, + w: 8, + h: 8, }, ) .expect("crop direct plan"); @@ -15346,10 +17498,10 @@ mod tests { } #[test] - fn cropped_region_ht_direct_plan_crops_all_idwt_levels() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { + fn cropped_region_ht_direct_plan_keeps_idwt_windows_bounded() { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { pixels.push(((x * 3 + y * 5) & 0xff) as u8); } } @@ -15361,7 +17513,7 @@ mod tests { ..EncodeOptions::default() }; let bytes = - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); + encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); let mut context = DecoderContext::default(); let plan = image @@ -15377,10 +17529,10 @@ mod tests { crop_prepared_direct_grayscale_plan_to_output_region( &mut prepared, signinum_core::Rect { - x: 56, - y: 56, - w: 16, - h: 16, + x: 112, + y: 112, + w: 32, + h: 32, }, ) .expect("crop direct plan"); @@ -15390,10 +17542,16 @@ mod tests { for (level_idx, (full_len, cropped_len)) in cropped_idwt_levels.iter().copied().enumerate() { assert!( - cropped_len > 0 && cropped_len < full_len, - "cropped ROI should reduce IDWT level {level_idx} output work; full={full_len}, cropped={cropped_len}" + cropped_len > 0 && cropped_len <= full_len, + "cropped ROI should keep IDWT level {level_idx} bounded; full={full_len}, cropped={cropped_len}" ); } + assert!( + cropped_idwt_levels + .iter() + .any(|(full_len, cropped_len)| cropped_len < full_len), + "cropped ROI should reduce at least one IDWT level" + ); } fn prepared_direct_grayscale_ht_job_count(plan: &super::PreparedDirectGrayscalePlan) -> usize { diff --git a/crates/signinum-j2k-metal/src/hybrid.rs b/crates/signinum-j2k-metal/src/hybrid.rs new file mode 100644 index 00000000..75d4e48a --- /dev/null +++ b/crates/signinum-j2k-metal/src/hybrid.rs @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(test)] +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{sync::Arc, time::Instant}; + +use metal::Device; +use rayon::prelude::*; +use signinum_core::{Downscale, PixelFormat, Rect}; +use signinum_j2k::J2kError; +use signinum_j2k_native::{ + DecodeSettings as NativeDecodeSettings, DecoderContext as NativeDecoderContext, + Image as NativeImage, +}; + +use crate::{direct, Error, J2kDecoder, Surface}; + +pub(crate) const RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED: &str = + "J2K Metal ROI+scaled hybrid decode currently supports single-tile RGB direct plans for Rgb8/Rgba8/Rgb16"; + +#[cfg(test)] +static REGION_SCALED_COLOR_PLAN_BUILDS: AtomicUsize = AtomicUsize::new(0); + +#[cfg(test)] +pub(crate) fn reset_region_scaled_color_plan_builds_for_test() { + REGION_SCALED_COLOR_PLAN_BUILDS.store(0, Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn region_scaled_color_plan_builds_for_test() -> usize { + REGION_SCALED_COLOR_PLAN_BUILDS.load(Ordering::Relaxed) +} + +enum PreparedRegionScaledDirectPlan { + Gray(crate::compute::PreparedDirectGrayscalePlan), + Color(crate::compute::PreparedDirectColorPlan), +} + +pub(crate) fn decode_region_scaled_direct_to_surface( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result, Error> { + let Some(prepared) = build_region_scaled_direct_plan(input, fmt, roi, scale)? else { + return Ok(None); + }; + execute_region_scaled_direct_plan(prepared, fmt) +} + +pub(crate) fn decode_region_scaled_direct_to_surface_with_device( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + device: &Device, +) -> Result, Error> { + let Some(prepared) = build_region_scaled_direct_plan(input, fmt, roi, scale)? else { + return Ok(None); + }; + execute_region_scaled_direct_plan_with_device(prepared, fmt, device) +} + +fn build_region_scaled_direct_plan( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result, Error> { + match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + match build_region_scaled_direct_gray_plan(input, roi, scale) { + Ok(plan) => Ok(Some(PreparedRegionScaledDirectPlan::Gray(plan))), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { + Ok(Some(PreparedRegionScaledDirectPlan::Color( + build_region_scaled_direct_color_plan(input, roi, scale)?, + ))) + } + _ => Ok(None), + } +} + +#[doc(hidden)] +pub(crate) fn benchmark_region_scaled_direct_plan_prepare( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result<(), Error> { + if build_region_scaled_direct_plan(input, fmt, roi, scale)?.is_some() { + Ok(()) + } else { + Err(Error::UnsupportedMetalRequest { + reason: "J2K MetalDirect ROI+scaled plan preparation is unsupported for this benchmark input", + }) + } +} + +fn execute_region_scaled_direct_plan( + plan: PreparedRegionScaledDirectPlan, + fmt: PixelFormat, +) -> Result, Error> { + match plan { + PreparedRegionScaledDirectPlan::Gray(plan) => { + match crate::compute::execute_prepared_direct_grayscale_plan(&plan, fmt) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + PreparedRegionScaledDirectPlan::Color(plan) => { + match crate::compute::execute_hybrid_cpu_tier1_direct_color_plan(&plan, fmt) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { + Err(Error::UnsupportedMetalRequest { + reason: RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED, + }) + } + Err(error) => Err(error), + } + } + } +} + +fn execute_region_scaled_direct_plan_with_device( + plan: PreparedRegionScaledDirectPlan, + fmt: PixelFormat, + device: &Device, +) -> Result, Error> { + match plan { + PreparedRegionScaledDirectPlan::Gray(plan) => { + match crate::compute::execute_prepared_direct_grayscale_plan_with_device( + &plan, fmt, device, + ) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + PreparedRegionScaledDirectPlan::Color(plan) => { + match crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_with_device( + &plan, fmt, device, + ) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { + Err(Error::UnsupportedMetalRequest { + reason: RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED, + }) + } + Err(error) => Err(error), + } + } + } +} + +pub(crate) fn decode_region_scaled_grayscale_batch_direct_to_device( + requests: &[(Arc<[u8]>, Rect, Downscale)], + fmt: PixelFormat, +) -> Result, Error> { + if requests.is_empty() { + return Ok(Vec::new()); + } + if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { + return Err(Error::MetalKernel { + message: format!( + "J2K MetalDirect region-scaled grayscale batch does not support {fmt:?}" + ), + }); + } + + let mut plans = Vec::with_capacity(requests.len()); + for (input, roi, scale) in requests { + let plan = build_region_scaled_direct_gray_plan(input.as_ref(), *roi, *scale)?; + plans.push(Arc::new(plan)); + } + crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) +} + +pub(crate) fn decode_region_scaled_color_batch_direct_to_device( + requests: &[(Arc<[u8]>, Rect, Downscale)], + fmt: PixelFormat, +) -> Result, Error> { + if requests.is_empty() { + return Ok(Vec::new()); + } + if !matches!( + fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) { + return Err(Error::MetalKernel { + message: format!("J2K MetalDirect region-scaled color batch does not support {fmt:?}"), + }); + } + + if let Some((input, roi, scale)) = repeated_region_scaled_request(requests) { + let plan = Arc::new(build_region_scaled_direct_color_plan( + input.as_ref(), + roi, + scale, + )?); + let plans = vec![plan; requests.len()]; + return crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt); + } + + let plans = requests + .par_iter() + .map(|(input, roi, scale)| { + build_region_scaled_direct_color_plan(input.as_ref(), *roi, *scale).map(Arc::new) + }) + .collect::, _>>()?; + crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt) +} + +fn repeated_region_scaled_request( + requests: &[(Arc<[u8]>, Rect, Downscale)], +) -> Option<(&Arc<[u8]>, Rect, Downscale)> { + let (first_input, first_roi, first_scale) = requests.first()?; + requests + .iter() + .all(|(input, roi, scale)| { + *roi == *first_roi + && *scale == *first_scale + && (Arc::ptr_eq(input, first_input) || input.as_ref() == first_input.as_ref()) + }) + .then_some((first_input, *first_roi, *first_scale)) +} + +fn build_region_scaled_direct_gray_plan( + input: &[u8], + roi: Rect, + scale: Downscale, +) -> Result { + let image = build_region_scaled_native_image(input, scale)?; + let mut context = NativeDecoderContext::default(); + let output_region = roi.scaled_covering(scale); + let plan = match image.build_direct_grayscale_plan_region_with_context( + &mut context, + ( + output_region.x, + output_region.y, + output_region.w, + output_region.h, + ), + ) { + Ok(plan) => plan, + Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { + return Err(Error::MetalKernel { + message: format!( + "explicit J2K MetalDirect region-scaled batch currently supports grayscale direct plans only: {error}" + ), + }); + } + Err(error) => { + return Err(Error::Decode(J2kError::Backend(format!( + "failed to build J2K MetalDirect region-scaled grayscale plan: {error}" + )))); + } + }; + let mut prepared = crate::compute::prepare_direct_grayscale_plan(&plan)?; + crate::compute::crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + output_region, + )?; + Ok(prepared) +} + +fn build_region_scaled_direct_color_plan( + input: &[u8], + roi: Rect, + scale: Downscale, +) -> Result { + #[cfg(test)] + REGION_SCALED_COLOR_PLAN_BUILDS.fetch_add(1, Ordering::Relaxed); + + let profile_stages = crate::compute::metal_profile_stages_enabled(); + let total_started = profile_stages.then(Instant::now); + let native_image_started = profile_stages.then(Instant::now); + let image = build_region_scaled_native_image(input, scale)?; + let native_image_us = native_image_started.map(elapsed_us).unwrap_or_default(); + let direct_plan_started = profile_stages.then(Instant::now); + let mut context = NativeDecoderContext::default(); + let output_region = roi.scaled_covering(scale); + let plan = match image.build_direct_color_plan_region_with_context( + &mut context, + ( + output_region.x, + output_region.y, + output_region.w, + output_region.h, + ), + ) { + Ok(plan) => plan, + Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { + return Err(Error::UnsupportedMetalRequest { + reason: RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED, + }); + } + Err(error) => { + return Err(Error::Decode(J2kError::Backend(format!( + "failed to build J2K MetalDirect region-scaled color plan: {error}" + )))); + } + }; + let direct_plan_us = direct_plan_started.map(elapsed_us).unwrap_or_default(); + let prepare_started = profile_stages.then(Instant::now); + let mut prepared = crate::compute::prepare_direct_color_plan_for_cpu_upload(&plan)?; + let prepare_us = prepare_started.map(elapsed_us).unwrap_or_default(); + let crop_started = profile_stages.then(Instant::now); + crate::compute::crop_prepared_direct_color_plan_to_output_region(&mut prepared, output_region)?; + let crop_us = crop_started.map(elapsed_us).unwrap_or_default(); + if let Some(started) = total_started { + emit_region_scaled_color_plan_build_timings( + native_image_us, + direct_plan_us, + prepare_us, + crop_us, + elapsed_us(started), + ); + } + Ok(prepared) +} + +fn elapsed_us(started: Instant) -> u128 { + started.elapsed().as_micros() +} + +fn emit_region_scaled_color_plan_build_timings( + native_image_us: u128, + direct_plan_us: u128, + prepare_us: u128, + crop_us: u128, + total_us: u128, +) { + if !crate::compute::metal_profile_stages_enabled() { + return; + } + + for (stage, elapsed_us) in [ + ("native_image", native_image_us), + ("direct_color_plan", direct_plan_us), + ("prepare_cpu_upload", prepare_us), + ("crop_prepared_plan", crop_us), + ("plan_total", total_us), + ] { + eprintln!( + "signinum_profile codec=j2k op=decode path=metal_direct_hybrid_plan stage={stage} fmt=Rgb batch_count=1 elapsed_us={elapsed_us}" + ); + } +} + +fn build_region_scaled_native_image( + input: &[u8], + scale: Downscale, +) -> Result, Error> { + let decoder = J2kDecoder::new(input)?; + let dims = decoder.inner.info().dimensions; + let target_dims = ( + dims.0.div_ceil(scale.denominator()), + dims.1.div_ceil(scale.denominator()), + ); + let settings = NativeDecodeSettings { + target_resolution: Some(target_dims), + ..NativeDecodeSettings::default() + }; + let image = + NativeImage::new(input, &settings).map_err(|error| J2kError::Backend(error.to_string()))?; + Ok(image) +} + +fn is_direct_region_scaled_runtime_fallback_error(error: &Error) -> bool { + crate::is_direct_runtime_fallback_error(error) +} diff --git a/crates/signinum-j2k-metal/src/idwt.metal b/crates/signinum-j2k-metal/src/idwt.metal index f5b431b1..5cb0fe91 100644 --- a/crates/signinum-j2k-metal/src/idwt.metal +++ b/crates/signinum-j2k-metal/src/idwt.metal @@ -149,19 +149,27 @@ kernel void j2k_idwt_interleave( if (low_y && low_x) { const uint band_x = full_band_x - params.ll_x; const uint band_y = full_band_y - params.ll_y; - out[out_idx] = ll[band_y * params.ll_width + band_x]; + out[out_idx] = (band_x < params.ll_width && band_y < params.ll_height) + ? ll[band_y * params.ll_width + band_x] + : 0.0f; } else if (low_y) { const uint band_x = full_band_x - params.hl_x; const uint band_y = full_band_y - params.hl_y; - out[out_idx] = hl[band_y * params.hl_width + band_x]; + out[out_idx] = (band_x < params.hl_width && band_y < params.hl_height) + ? hl[band_y * params.hl_width + band_x] + : 0.0f; } else if (low_x) { const uint band_x = full_band_x - params.lh_x; const uint band_y = full_band_y - params.lh_y; - out[out_idx] = lh[band_y * params.lh_width + band_x]; + out[out_idx] = (band_x < params.lh_width && band_y < params.lh_height) + ? lh[band_y * params.lh_width + band_x] + : 0.0f; } else { const uint band_x = full_band_x - params.hh_x; const uint band_y = full_band_y - params.hh_y; - out[out_idx] = hh[band_y * params.hh_width + band_x]; + out[out_idx] = (band_x < params.hh_width && band_y < params.hh_height) + ? hh[band_y * params.hh_width + band_x] + : 0.0f; } } @@ -192,19 +200,27 @@ kernel void j2k_idwt_interleave_batched( if (low_y && low_x) { const uint band_x = full_band_x - params.ll_x; const uint band_y = full_band_y - params.ll_y; - out[out_idx] = ll[gid.z * params.ll_instance_stride + band_y * params.ll_width + band_x]; + out[out_idx] = (band_x < params.ll_width && band_y < params.ll_height) + ? ll[gid.z * params.ll_instance_stride + band_y * params.ll_width + band_x] + : 0.0f; } else if (low_y) { const uint band_x = full_band_x - params.hl_x; const uint band_y = full_band_y - params.hl_y; - out[out_idx] = hl[gid.z * params.hl_instance_stride + band_y * params.hl_width + band_x]; + out[out_idx] = (band_x < params.hl_width && band_y < params.hl_height) + ? hl[gid.z * params.hl_instance_stride + band_y * params.hl_width + band_x] + : 0.0f; } else if (low_x) { const uint band_x = full_band_x - params.lh_x; const uint band_y = full_band_y - params.lh_y; - out[out_idx] = lh[gid.z * params.lh_instance_stride + band_y * params.lh_width + band_x]; + out[out_idx] = (band_x < params.lh_width && band_y < params.lh_height) + ? lh[gid.z * params.lh_instance_stride + band_y * params.lh_width + band_x] + : 0.0f; } else { const uint band_x = full_band_x - params.hh_x; const uint band_y = full_band_y - params.hh_y; - out[out_idx] = hh[gid.z * params.hh_instance_stride + band_y * params.hh_width + band_x]; + out[out_idx] = (band_x < params.hh_width && band_y < params.hh_height) + ? hh[gid.z * params.hh_instance_stride + band_y * params.hh_width + band_x] + : 0.0f; } } @@ -461,39 +477,27 @@ kernel void j2k_idwt_irreversible97_single_decomposition( if (low_y && low_x) { const uint band_x = full_band_x - params.ll_x; const uint band_y = full_band_y - params.ll_y; - if (band_x >= params.ll_width || band_y >= params.ll_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 2u; - return; - } - out[out_idx] = ll[band_y * params.ll_width + band_x]; + out[out_idx] = (band_x < params.ll_width && band_y < params.ll_height) + ? ll[band_y * params.ll_width + band_x] + : 0.0f; } else if (low_y) { const uint band_x = full_band_x - params.hl_x; const uint band_y = full_band_y - params.hl_y; - if (band_x >= params.hl_width || band_y >= params.hl_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 3u; - return; - } - out[out_idx] = hl[band_y * params.hl_width + band_x]; + out[out_idx] = (band_x < params.hl_width && band_y < params.hl_height) + ? hl[band_y * params.hl_width + band_x] + : 0.0f; } else if (low_x) { const uint band_x = full_band_x - params.lh_x; const uint band_y = full_band_y - params.lh_y; - if (band_x >= params.lh_width || band_y >= params.lh_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 4u; - return; - } - out[out_idx] = lh[band_y * params.lh_width + band_x]; + out[out_idx] = (band_x < params.lh_width && band_y < params.lh_height) + ? lh[band_y * params.lh_width + band_x] + : 0.0f; } else { const uint band_x = full_band_x - params.hh_x; const uint band_y = full_band_y - params.hh_y; - if (band_x >= params.hh_width || band_y >= params.hh_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 5u; - return; - } - out[out_idx] = hh[band_y * params.hh_width + band_x]; + out[out_idx] = (band_x < params.hh_width && band_y < params.hh_height) + ? hh[band_y * params.hh_width + band_x] + : 0.0f; } } } diff --git a/crates/signinum-j2k-metal/src/lib.rs b/crates/signinum-j2k-metal/src/lib.rs index bb6487c0..68d701fe 100644 --- a/crates/signinum-j2k-metal/src/lib.rs +++ b/crates/signinum-j2k-metal/src/lib.rs @@ -14,6 +14,8 @@ mod direct; mod encode; #[cfg(any(test, target_os = "macos"))] mod ht; +#[cfg(target_os = "macos")] +mod hybrid; #[cfg(any(test, target_os = "macos"))] mod idwt; #[cfg(any(test, target_os = "macos"))] @@ -52,6 +54,8 @@ use signinum_j2k_native::{ #[cfg(target_os = "macos")] use metal::{Buffer, Device, MTLResourceOptions}; +#[doc(hidden)] +pub use batch::{benchmark_group_region_scaled_requests, BenchmarkGroupedRequests}; #[doc(hidden)] pub use dicom::{ extract_dicom_encapsulated_frames, extract_dicom_encapsulated_frames_with_limit, @@ -84,6 +88,28 @@ pub use encode::{ SubmittedJ2kLosslessMetalEncodeBatch, }; +#[cfg(target_os = "macos")] +#[doc(hidden)] +pub fn benchmark_region_scaled_direct_plan_prepare( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result<(), Error> { + hybrid::benchmark_region_scaled_direct_plan_prepare(input, fmt, roi, scale) +} + +#[cfg(not(target_os = "macos"))] +#[doc(hidden)] +pub fn benchmark_region_scaled_direct_plan_prepare( + _input: &[u8], + _fmt: PixelFormat, + _roi: Rect, + _scale: Downscale, +) -> Result<(), Error> { + Err(Error::MetalUnavailable) +} + #[derive(Debug, thiserror::Error)] pub enum Error { #[error(transparent)] @@ -180,7 +206,6 @@ pub enum SurfaceResidency { #[cfg(target_os = "macos")] const CPU_STAGED_METAL_REQUIRES_EXPLICIT_API: &str = "CPU-staged Metal upload requires the explicit CPU-staged API; BackendRequest::Metal only accepts resident Metal decode"; - impl Surface { pub fn residency(&self) -> SurfaceResidency { self.residency @@ -950,21 +975,7 @@ impl<'a> J2kDecoder<'a> { roi: Rect, scale: Downscale, ) -> Result, Error> { - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - return Ok(None); - } - let prepared = match build_region_scaled_direct_gray_plan(self.inner.bytes(), roi, scale) { - Ok(prepared) => prepared, - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { - return Ok(None); - } - Err(error) => return Err(error), - }; - match crate::compute::execute_prepared_direct_grayscale_plan(&Arc::new(prepared), fmt) { - Ok(surface) => Ok(Some(surface)), - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), - Err(error) => Err(error), - } + crate::hybrid::decode_region_scaled_direct_to_surface(self.inner.bytes(), fmt, roi, scale) } #[cfg(target_os = "macos")] @@ -975,25 +986,13 @@ impl<'a> J2kDecoder<'a> { scale: Downscale, device: &Device, ) -> Result, Error> { - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - return Ok(None); - } - let prepared = match build_region_scaled_direct_gray_plan(self.inner.bytes(), roi, scale) { - Ok(prepared) => prepared, - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { - return Ok(None); - } - Err(error) => return Err(error), - }; - match crate::compute::execute_prepared_direct_grayscale_plan_with_device( - &Arc::new(prepared), + crate::hybrid::decode_region_scaled_direct_to_surface_with_device( + self.inner.bytes(), fmt, + roi, + scale, device, - ) { - Ok(surface) => Ok(Some(surface)), - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), - Err(error) => Err(error), - } + ) } #[cfg(target_os = "macos")] @@ -1454,18 +1453,6 @@ impl<'a> J2kDecoder<'a> { if let Some(error) = routing::decision_error(route) { return Err(error); } - #[cfg(target_os = "macos")] - { - if matches!(route, routing::RouteDecision::MetalKernel) - && !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) - { - return Err(Error::UnsupportedMetalRequest { - reason: - "J2K Metal ROI+scaled direct decode currently supports Gray8/Gray16 only", - }); - } - } - let plan = DeviceDecodePlan::for_image( self.inner.info().dimensions, DeviceDecodeRequest::RegionScaled { roi, scale }, @@ -1500,7 +1487,7 @@ impl<'a> J2kDecoder<'a> { } if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { return Err(Error::UnsupportedMetalRequest { - reason: "J2K Metal ROI+scaled direct decode currently supports Gray8/Gray16 only", + reason: "J2K Metal session scaled decode currently supports Gray8/Gray16 only", }); } @@ -1656,14 +1643,10 @@ fn is_direct_runtime_fallback_error(error: &Error) -> bool { || message.contains("unsupported HT kernel input") || message.contains("direct component plan") || message.contains("currently supports grayscale direct plans only") + || message.contains("currently supports color direct plans only") ) } -#[cfg(target_os = "macos")] -fn is_direct_region_scaled_runtime_fallback_error(error: &Error) -> bool { - is_direct_runtime_fallback_error(error) -} - #[cfg(target_os = "macos")] pub(crate) fn decode_full_grayscale_batch_direct_to_device( inputs: &[Arc<[u8]>], @@ -1693,72 +1676,6 @@ pub(crate) fn decode_full_grayscale_batch_direct_to_device( crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) } -#[cfg(target_os = "macos")] -pub(crate) fn decode_region_scaled_grayscale_batch_direct_to_device( - requests: &[(Arc<[u8]>, Rect, Downscale)], - fmt: PixelFormat, -) -> Result, Error> { - if requests.is_empty() { - return Ok(Vec::new()); - } - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - return Err(Error::MetalKernel { - message: format!( - "J2K MetalDirect region-scaled grayscale batch does not support {fmt:?}" - ), - }); - } - - let mut plans = Vec::with_capacity(requests.len()); - for (input, roi, scale) in requests { - let plan = build_region_scaled_direct_gray_plan(input.as_ref(), *roi, *scale)?; - plans.push(Arc::new(plan)); - } - crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) -} - -#[cfg(target_os = "macos")] -fn build_region_scaled_direct_gray_plan( - input: &[u8], - roi: Rect, - scale: Downscale, -) -> Result { - let decoder = J2kDecoder::new(input)?; - let dims = decoder.inner.info().dimensions; - let target_dims = ( - dims.0.div_ceil(scale.denominator()), - dims.1.div_ceil(scale.denominator()), - ); - let settings = NativeDecodeSettings { - target_resolution: Some(target_dims), - ..NativeDecodeSettings::default() - }; - let image = - NativeImage::new(input, &settings).map_err(|error| J2kError::Backend(error.to_string()))?; - let mut context = NativeDecoderContext::default(); - let plan = match image.build_direct_grayscale_plan_with_context(&mut context) { - Ok(plan) => plan, - Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { - return Err(Error::MetalKernel { - message: format!( - "explicit J2K MetalDirect region-scaled batch currently supports grayscale direct plans only: {error}" - ), - }); - } - Err(error) => { - return Err(Error::Decode(J2kError::Backend(format!( - "failed to build J2K MetalDirect region-scaled grayscale plan: {error}" - )))); - } - }; - let mut prepared = crate::compute::prepare_direct_grayscale_plan(&plan)?; - crate::compute::crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - roi.scaled_covering(scale), - )?; - Ok(prepared) -} - #[cfg(target_os = "macos")] pub(crate) fn decode_full_color_batch_direct_to_device( inputs: &[Arc<[u8]>], @@ -2180,4 +2097,44 @@ mod tests { && reason.contains("Metal") )); } + + #[cfg(target_os = "macos")] + #[test] + fn repeated_region_scaled_color_batch_reuses_prepared_plan() { + if Device::system_default().is_none() { + eprintln!("skipping repeated color plan reuse test: no Metal device"); + return; + } + + let pixels = signinum_test_support::gradient_u8(64, 64, 3); + let options = signinum_j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..signinum_j2k_native::EncodeOptions::default() + }; + let input = Arc::<[u8]>::from( + signinum_j2k_native::encode(&pixels, 64, 64, 3, 8, false, &options) + .expect("encode rgb8"), + ); + let roi = Rect { + x: 8, + y: 8, + w: 32, + h: 32, + }; + let scale = Downscale::Quarter; + let requests = vec![(input.clone(), roi, scale); 4]; + hybrid::reset_region_scaled_color_plan_builds_for_test(); + + let surfaces = + hybrid::decode_region_scaled_color_batch_direct_to_device(&requests, PixelFormat::Rgb8) + .expect("repeated RGB region-scaled batch"); + + assert_eq!(surfaces.len(), requests.len()); + assert_eq!( + hybrid::region_scaled_color_plan_builds_for_test(), + 1, + "repeated RGB ROI+scaled batches should build and crop one prepared direct color plan" + ); + } } diff --git a/crates/signinum-j2k-metal/src/store.metal b/crates/signinum-j2k-metal/src/store.metal index 316a9875..5f84fbf6 100644 --- a/crates/signinum-j2k-metal/src/store.metal +++ b/crates/signinum-j2k-metal/src/store.metal @@ -18,6 +18,7 @@ struct J2kStoreParams { struct J2kRepeatedStoreParams { uint input_width; uint input_height; + uint input_instance_stride; uint source_x; uint source_y; uint copy_width; @@ -93,14 +94,13 @@ kernel void j2k_store_component_repeated( return; } - const uint input_plane_len = params.input_width * params.input_height; const uint output_plane_len = params.output_width * params.output_height; const uint src_x = params.source_x + gid.x; const uint src_y = params.source_y + gid.y; const uint dst_x = params.output_x + gid.x; const uint dst_y = params.output_y + gid.y; - const uint src_idx = gid.z * input_plane_len + src_y * params.input_width + src_x; + const uint src_idx = gid.z * params.input_instance_stride + src_y * params.input_width + src_x; const uint dst_idx = gid.z * output_plane_len + dst_y * params.output_width + dst_x; output[dst_idx] = input[src_idx] + params.addend; } diff --git a/crates/signinum-j2k-metal/tests/bench_harness.rs b/crates/signinum-j2k-metal/tests/bench_harness.rs index 213386e1..ee1c966e 100644 --- a/crates/signinum-j2k-metal/tests/bench_harness.rs +++ b/crates/signinum-j2k-metal/tests/bench_harness.rs @@ -99,11 +99,12 @@ fn compare_bench_keeps_multiple_htj2k_gray_sizes() { common.contains("inputs.extend(ht_bench_inputs()"), "bench_inputs must include all generated HTJ2K size candidates, not only the first success" ); - assert!( - common.contains("(\"htj2k_gray_1024\", 1024_u32, 1024_u32)") - && common.contains("(\"htj2k_gray_512\", 512_u32, 512_u32)"), - "HTJ2K bench coverage must include 1024 and 512 grayscale tiles" - ); + for fixture_name in ["\"htj2k_gray_1024\"", "\"htj2k_gray_512\""] { + assert!( + common.contains(fixture_name), + "HTJ2K bench coverage must include {fixture_name}" + ); + } } #[test] @@ -125,7 +126,7 @@ fn external_wsi_bench_groups_by_codec_family() { } #[test] -fn metal_region_scaled_benches_gate_to_grayscale_direct_modes() { +fn metal_region_scaled_benches_gate_to_direct_modes() { let common = include_str!("../benches/common/mod.rs"); assert!( @@ -133,8 +134,8 @@ fn metal_region_scaled_benches_gate_to_grayscale_direct_modes() { "ROI+scaled Metal benchmark support must have a cheap format gate" ); assert!( - common.contains("matches!(mode, DecodeMode::Gray8 | DecodeMode::Gray16)"), - "ROI+scaled Metal benchmarks must only advertise direct grayscale modes until RGB is GPU-native" + common.contains("DecodeMode::Gray8 | DecodeMode::Gray16 | DecodeMode::Rgb8"), + "ROI+scaled Metal benchmarks must advertise RGB once RGB is GPU-native" ); for helper in [ "signinum_metal_supports_region_scaled", @@ -154,3 +155,75 @@ fn metal_region_scaled_benches_gate_to_grayscale_direct_modes() { ); } } + +#[test] +fn compare_bench_rgb_region_scaled_reports_resident_vs_cpu_staged_metal() { + let common = include_str!("../benches/common/mod.rs"); + let compare = include_str!("../benches/compare.rs"); + + assert!( + common.contains("signinum_cpu_staged_metal_decode_tile_batch_region_scaled"), + "common benchmarks must expose a CPU decode plus CPU-to-Metal upload baseline" + ); + assert!( + common.contains("decode_region_scaled_to_cpu_staged_metal_surface_with_session"), + "CPU-staged baseline must use the explicit CPU-staged Metal API" + ); + assert!( + compare.contains("wsi_tile_batch_region_scaled_rgb_q4"), + "compare bench must include an RGB ROI+scaled batch group" + ); + assert!( + compare.contains("signinum-cpu-staged-metal") + && compare.contains("signinum-metal-resident"), + "RGB ROI+scaled benches must report CPU-staged Metal and resident hybrid Metal separately" + ); +} + +#[test] +fn compare_bench_distinct_rgb_region_scaled_reports_cpu_auto_and_resident_metal() { + let compare = include_str!("../benches/compare.rs"); + + assert!( + compare.contains("wsi_tile_batch_region_scaled_rgb_distinct_q4"), + "compare bench must include a distinct RGB ROI+scaled batch group" + ); + assert!( + compare.contains("distinct_rgb_tile_batch_inputs(input, count)"), + "distinct RGB ROI+scaled group must use generated distinct RGB tiles" + ); + assert!( + compare.contains("signinum_decode_tile_batch_region_scaled_distinct") + && compare.contains("signinum_adaptive_decode_tile_batch_region_scaled_distinct") + && compare.contains("signinum_metal_decode_tile_batch_region_scaled_distinct"), + "distinct RGB ROI+scaled group must report CPU, Auto, and explicit resident Metal timings" + ); +} + +#[test] +fn compare_bench_exposes_htj2k_measurement_first_surfaces() { + let common = include_str!("../benches/common/mod.rs"); + let compare = include_str!("../benches/compare.rs"); + + for expected in [ + "htj2k_region_scaled_plan_build", + "htj2k_feeder_coalesce", + "htj2k_metal_route", + ] { + assert!( + compare.contains(expected), + "compare benchmark is missing `{expected}`" + ); + } + + for expected in [ + "benchmark_region_scaled_direct_plan_prepare", + "benchmark_group_region_scaled_requests", + "\"htj2k_rgb_512\"", + ] { + assert!( + common.contains(expected), + "common benchmark helpers are missing `{expected}`" + ); + } +} diff --git a/crates/signinum-j2k-metal/tests/device.rs b/crates/signinum-j2k-metal/tests/device.rs index 17bb9808..1398e90a 100644 --- a/crates/signinum-j2k-metal/tests/device.rs +++ b/crates/signinum-j2k-metal/tests/device.rs @@ -66,6 +66,24 @@ fn fixture_ht_gray8_sized(width: u32, height: u32) -> Vec { encode_htj2k(&pixels, width, height, 1, 8, false, &options).expect("encode sized ht gray8") } +fn fixture_rgb8_sized(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xFF) as u8); + pixels.push(((x * 7 + y * 11 + 13) & 0xFF) as u8); + pixels.push(((x * 17 + y * 19 + 29) & 0xFF) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + guard_bits: 2, + ..EncodeOptions::default() + }; + encode(&pixels, width, height, 3, 8, false, &options).expect("encode sized rgb8") +} + fn fixture_ht_gray8_unsupported_direct_width() -> Vec { let width = 512u32; let height = 8u32; @@ -1209,6 +1227,64 @@ fn submitted_auto_region_scaled_ht_grayscale_1024_batch16_uses_one_metal_batch() ); } +#[test] +fn submitted_auto_region_scaled_rgb_1024_batch16_uses_hybrid_metal() { + let bytes = fixture_rgb8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut ctx = signinum_core::DecoderContext::::new(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + let submissions = (0..16) + .map(|_| { + Codec::submit_tile_region_scaled_to_device( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Rgb8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("submit auto region-scaled RGB tile") + }) + .collect::>(); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host region-scaled RGB decode"); + + for submission in submissions { + let surface = submission.wait().expect("auto region-scaled RGB surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); + } + assert_eq!( + session.submissions(), + 1, + "1024-class auto ROI+scaled RGB tile batches should use one resident hybrid Metal batch" + ); +} + #[test] fn submitted_auto_region_scaled_ht_grayscale_batch16_is_not_order_dependent() { let small_bytes = fixture_ht_gray8_sized(64, 64); @@ -1572,6 +1648,53 @@ fn explicit_metal_region_scaled_grayscale_matches_host_decode() { assert_eq!(surface.as_bytes(), host.as_slice()); } +#[test] +fn explicit_metal_region_scaled_grayscale_large_cropped_matches_host_decode() { + let bytes = fixture_gray8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 512, + }; + + for scale in [Downscale::Half, Downscale::None] { + let scaled = roi.scaled_covering(scale); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region scaled decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + if surface.as_bytes() != host.as_slice() { + let mismatch = surface + .as_bytes() + .iter() + .zip(&host) + .position(|(actual, expected)| actual != expected) + .expect("mismatched buffers should have a differing byte"); + panic!( + "scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", + surface.as_bytes()[mismatch], + host[mismatch] + ); + } + } +} + #[test] fn explicit_metal_region_scaled_htj2k_grayscale_matches_host_decode() { let bytes = fixture_ht_gray8(); @@ -1641,28 +1764,145 @@ fn explicit_metal_region_scaled_htj2k_falls_back_when_direct_width_is_unsupporte } #[test] -fn explicit_metal_region_scaled_rgb_is_rejected_instead_of_cpu_staging() { - let bytes = fixture_rgb8(); +fn explicit_metal_region_scaled_rgb_matches_host_decode() { + let bytes = fixture_direct_rgb8_variant(3); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host region scaled RGB decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("rgba8 host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgba8, + roi, + scale, + ) + .expect("host region scaled RGBA decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("rgba8 decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgba8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGBA decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); + + let bytes = fixture_rgb12(); let roi = Rect { x: 0, y: 0, - w: 1, + w: 2, h: 1, }; - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let Err(error) = decoder.decode_region_scaled_to_device( - PixelFormat::Rgb8, - roi, - Downscale::None, - BackendRequest::Metal, - ) else { - panic!("explicit Metal RGB ROI+scaled should not hide a CPU-staged path") + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut host_decoder = J2kDecoder::new(&bytes).expect("rgb16 host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb16, + roi, + scale, + ) + .expect("host region scaled RGB16 decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("rgb16 decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb16, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB16 decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); +} + +#[test] +fn explicit_metal_region_scaled_rgb_large_cropped_matches_host_decode() { + let bytes = fixture_rgb8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 512, }; - assert!(matches!( - error, - Error::UnsupportedMetalRequest { reason } - if reason.contains("ROI+scaled") && reason.contains("Gray8/Gray16") - )); + + for scale in [Downscale::Half, Downscale::None] { + let scaled = roi.scaled_covering(scale); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8] { + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * fmt.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + fmt, + roi, + scale, + ) + .expect("host region scaled RGB decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(fmt, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + if surface.as_bytes() != host.as_slice() { + let mismatch = surface + .as_bytes() + .iter() + .zip(&host) + .position(|(actual, expected)| actual != expected) + .expect("mismatched buffers should have a differing byte"); + panic!( + "fmt={fmt:?} scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", + surface.as_bytes()[mismatch], + host[mismatch] + ); + } + } + } } #[test] @@ -1756,3 +1996,21 @@ fn explicit_metal_tile_unsupported_rgba16_is_rejected() { ), } } + +#[test] +fn hybrid_ht_cpuupload_uses_worker_local_decode_workspace() { + let source = include_str!("../src/compute.rs"); + + assert!( + source.contains("decode_prepared_ht_jobs_on_cpu_with_workspace"), + "HT CPUUpload decode must expose a workspace-aware helper" + ); + assert!( + source.contains("HtCodeBlockDecodeWorkspace::default()"), + "parallel HT CPUUpload decode must initialize worker-local HT decode workspaces" + ); + assert!( + source.contains("decode_ht_code_block_scalar_with_workspace"), + "HT CPUUpload decode must call the scratch-reusing scalar helper" + ); +} diff --git a/crates/signinum-j2k-native/Cargo.toml b/crates/signinum-j2k-native/Cargo.toml index d4609f2b..02c9c566 100644 --- a/crates/signinum-j2k-native/Cargo.toml +++ b/crates/signinum-j2k-native/Cargo.toml @@ -33,6 +33,14 @@ criterion = { workspace = true } name = "tier1_bitplane" harness = false +[[bench]] +name = "htj2k_sigprop_phase" +harness = false + +[[bench]] +name = "direct_cpu" +harness = false + [lints.rust] unsafe_code = "forbid" unreachable_pub = "warn" diff --git a/crates/signinum-j2k-native/benches/direct_cpu.rs b/crates/signinum-j2k-native/benches/direct_cpu.rs new file mode 100644 index 00000000..b97f2c46 --- /dev/null +++ b/crates/signinum-j2k-native/benches/direct_cpu.rs @@ -0,0 +1,119 @@ +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use signinum_j2k_native::{ + encode_htj2k, execute_direct_color_plan_rgb8_into, execute_direct_color_plan_rgba8_into, + DecodeSettings, DecoderContext, EncodeOptions, Image, J2kDirectCpuScratch, J2kRect, +}; + +const TILE_SIDE: u32 = 512; + +fn patterned_rgb8(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + pixels.push(((x * 7 + y * 11 + 17) & 0xff) as u8); + pixels.push(((x * 13 + y * 19 + 31) & 0xff) as u8); + } + } + pixels +} + +fn htj2k_rgb_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_rgb8(width, height); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 8, false, &options).expect("encode HTJ2K RGB") +} + +fn direct_roi_plan(bytes: &[u8]) -> (signinum_j2k_native::J2kDirectColorPlan, J2kRect) { + let image = Image::new( + bytes, + &DecodeSettings { + target_resolution: Some((TILE_SIDE / 4, TILE_SIDE / 4)), + ..DecodeSettings::default() + }, + ) + .expect("scaled HTJ2K image"); + let output_region = J2kRect { + x0: 32, + y0: 32, + x1: 96, + y1: 96, + }; + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_region_with_context( + &mut context, + ( + output_region.x0, + output_region.y0, + output_region.width(), + output_region.height(), + ), + ) + .expect("direct RGB region plan"); + (plan, output_region) +} + +fn bench_direct_color_plan(c: &mut Criterion) { + let codestream = htj2k_rgb_codestream(TILE_SIDE, TILE_SIDE); + let (plan, output_region) = direct_roi_plan(&codestream); + let rgb_stride = output_region.width() as usize * 3; + let rgba_stride = output_region.width() as usize * 4; + let rgb_len = rgb_stride * output_region.height() as usize; + let rgba_len = rgba_stride * output_region.height() as usize; + + let mut group = c.benchmark_group("j2k_native_direct_cpu_color_plan"); + group.bench_function("htj2k_rgb8_roi256_q4_fresh_scratch", |b| { + b.iter(|| { + let mut scratch = J2kDirectCpuScratch::new(); + let mut out = vec![0_u8; rgb_len]; + execute_direct_color_plan_rgb8_into( + black_box(&plan), + output_region, + &mut scratch, + &mut out, + rgb_stride, + ) + .expect("execute RGB direct plan"); + black_box(out); + }); + }); + group.bench_function("htj2k_rgb8_roi256_q4_reuse_scratch", |b| { + let mut scratch = J2kDirectCpuScratch::new(); + let mut out = vec![0_u8; rgb_len]; + b.iter(|| { + execute_direct_color_plan_rgb8_into( + black_box(&plan), + output_region, + &mut scratch, + &mut out, + rgb_stride, + ) + .expect("execute RGB direct plan"); + black_box(&out); + }); + }); + group.bench_function("htj2k_rgba8_roi256_q4_reuse_scratch", |b| { + let mut scratch = J2kDirectCpuScratch::new(); + let mut out = vec![0_u8; rgba_len]; + b.iter(|| { + execute_direct_color_plan_rgba8_into( + black_box(&plan), + output_region, + &mut scratch, + &mut out, + rgba_stride, + ) + .expect("execute RGBA direct plan"); + black_box(&out); + }); + }); + group.finish(); +} + +criterion_group!(benches, bench_direct_color_plan); +criterion_main!(benches); diff --git a/crates/signinum-j2k-native/benches/htj2k_sigprop_phase.rs b/crates/signinum-j2k-native/benches/htj2k_sigprop_phase.rs new file mode 100644 index 00000000..938b3ff4 --- /dev/null +++ b/crates/signinum-j2k-native/benches/htj2k_sigprop_phase.rs @@ -0,0 +1,172 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use signinum_j2k_native::{ + decode_ht_code_block_scalar, decode_ht_sigprop_benchmark_state, + prepare_ht_sigprop_benchmark_state, DecodeSettings, DecoderContext, HtCodeBlockDecodeJob, + HtCodeBlockDecoder, Image, Result, +}; + +const HTJ2K_REFINEMENT_FIXTURE: &[u8] = + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k"); + +#[derive(Clone)] +struct OwnedHtCodeBlockDecodeJob { + data: Vec, + cleanup_length: u32, + refinement_length: u32, + width: u32, + height: u32, + missing_bit_planes: u8, + number_of_coding_passes: u8, + num_bitplanes: u8, + stripe_causal: bool, + strict: bool, + dequantization_step: f32, +} + +impl OwnedHtCodeBlockDecodeJob { + fn from_job(job: HtCodeBlockDecodeJob<'_>) -> Self { + Self { + data: job.data.to_vec(), + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + } + } + + fn borrowed_with_passes( + &self, + number_of_coding_passes: u8, + refinement_length: u32, + ) -> HtCodeBlockDecodeJob<'_> { + HtCodeBlockDecodeJob { + data: &self.data, + cleanup_length: self.cleanup_length, + refinement_length, + width: self.width, + height: self.height, + output_stride: self.width as usize, + missing_bit_planes: self.missing_bit_planes, + number_of_coding_passes, + num_bitplanes: self.num_bitplanes, + stripe_causal: self.stripe_causal, + strict: self.strict, + dequantization_step: self.dequantization_step, + } + } +} + +fn decode_scalar_ht_batch( + jobs: &[OwnedHtCodeBlockDecodeJob], + outputs: &mut [Vec], +) -> Result<()> { + for (job, output) in jobs.iter().zip(outputs.iter_mut()) { + decode_ht_code_block_scalar( + job.borrowed_with_passes(job.number_of_coding_passes, job.refinement_length), + output, + )?; + } + Ok(()) +} + +#[derive(Default)] +struct CollectingHtDecoder { + jobs: Vec, +} + +impl HtCodeBlockDecoder for CollectingHtDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + if job.refinement_length > 0 { + self.jobs.push(OwnedHtCodeBlockDecodeJob::from_job(job)); + } + + decode_ht_code_block_scalar(job, output) + } +} + +fn collect_refinement_fixture_jobs() -> Vec { + let image = + Image::new(HTJ2K_REFINEMENT_FIXTURE, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let mut decoder = CollectingHtDecoder::default(); + image + .decode_components_with_ht_decoder(&mut context, &mut decoder) + .expect("decode fixture while collecting HTJ2K jobs"); + assert_eq!( + decoder.jobs.len(), + 14, + "expected every ds0_ht_09_b11 HT block to carry refinement data" + ); + decoder.jobs +} + +fn bench_htj2k_refinement_sigprop_phase(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_refinement_sigprop_phase"); + let jobs = collect_refinement_fixture_jobs(); + let job = jobs + .first() + .expect("fixture must contain a refinement HTJ2K block"); + let sigprop_job = job.borrowed_with_passes(2, job.refinement_length); + let mut state = + prepare_ht_sigprop_benchmark_state(sigprop_job).expect("prepare SigProp benchmark state"); + let mut output = vec![0u32; state.output_len()]; + + group.bench_function("ds0_ht_09_b11_sigprop_only", |b| { + b.iter(|| { + decode_ht_sigprop_benchmark_state(black_box(&mut state), &mut output) + .expect("decode HTJ2K SigProp phase"); + black_box(&output); + }); + }); + + group.finish(); +} + +fn bench_htj2k_cpuupload_decode_batch(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cpuupload_decode_batch"); + let fixture_jobs = collect_refinement_fixture_jobs(); + + for count in [1_usize, 2, 4, fixture_jobs.len()] { + let jobs = fixture_jobs + .iter() + .cloned() + .cycle() + .take(count) + .collect::>(); + let mut outputs = jobs + .iter() + .map(|job| vec![0.0_f32; job.width as usize * job.height as usize]) + .collect::>(); + + group.bench_with_input( + BenchmarkId::new("ds0_ht_09_b11_scalar_batch", count), + &count, + |b, _| { + b.iter(|| { + decode_scalar_ht_batch(black_box(&jobs), black_box(&mut outputs)) + .expect("decode HTJ2K scalar batch"); + black_box(&outputs); + }); + }, + ); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_htj2k_refinement_sigprop_phase, + bench_htj2k_cpuupload_decode_batch +); +criterion_main!(benches); diff --git a/crates/signinum-j2k-native/benches/tier1_bitplane.rs b/crates/signinum-j2k-native/benches/tier1_bitplane.rs index 47d6f5ba..abbf3cd8 100644 --- a/crates/signinum-j2k-native/benches/tier1_bitplane.rs +++ b/crates/signinum-j2k-native/benches/tier1_bitplane.rs @@ -1,10 +1,109 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use rayon::prelude::*; +use rayon::ThreadPoolBuilder; use signinum_j2k_native::{ - decode_ht_code_block_scalar, decode_j2k_code_block_scalar, encode_ht_code_block_scalar, - encode_j2k_code_block_scalar_with_style, HtCodeBlockDecodeJob, J2kCodeBlockDecodeJob, - J2kCodeBlockStyle, J2kSubBandType, + collect_ht_cleanup_encode_distribution, decode_ht_code_block_scalar, + decode_j2k_code_block_scalar, encode_ht_code_block_scalar, + encode_j2k_code_block_scalar_with_style, DecodeSettings, DecoderContext, HtCodeBlockDecodeJob, + HtCodeBlockDecoder, Image, J2kCodeBlockDecodeJob, J2kCodeBlockStyle, J2kSubBandType, Result, }; +const HTJ2K_REFINEMENT_FIXTURE: &[u8] = + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k"); + +#[derive(Clone)] +struct OwnedHtCodeBlockDecodeJob { + data: Vec, + cleanup_length: u32, + refinement_length: u32, + width: u32, + height: u32, + missing_bit_planes: u8, + number_of_coding_passes: u8, + num_bitplanes: u8, + stripe_causal: bool, + strict: bool, + dequantization_step: f32, +} + +impl OwnedHtCodeBlockDecodeJob { + fn from_job(job: HtCodeBlockDecodeJob<'_>) -> Self { + Self { + data: job.data.to_vec(), + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + } + } + + fn borrowed_with_passes( + &self, + number_of_coding_passes: u8, + refinement_length: u32, + ) -> HtCodeBlockDecodeJob<'_> { + HtCodeBlockDecodeJob { + data: &self.data, + cleanup_length: self.cleanup_length, + refinement_length, + width: self.width, + height: self.height, + output_stride: self.width as usize, + missing_bit_planes: self.missing_bit_planes, + number_of_coding_passes, + num_bitplanes: self.num_bitplanes, + stripe_causal: self.stripe_causal, + strict: self.strict, + dequantization_step: self.dequantization_step, + } + } + + fn output_len(&self) -> usize { + self.width as usize * self.height as usize + } +} + +#[derive(Default)] +struct CollectingHtDecoder { + jobs: Vec, +} + +impl HtCodeBlockDecoder for CollectingHtDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + if job.refinement_length > 0 { + self.jobs.push(OwnedHtCodeBlockDecodeJob::from_job(job)); + } + + decode_ht_code_block_scalar(job, output) + } +} + +fn collect_refinement_fixture_jobs() -> Vec { + let image = + Image::new(HTJ2K_REFINEMENT_FIXTURE, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let mut decoder = CollectingHtDecoder::default(); + image + .decode_components_with_ht_decoder(&mut context, &mut decoder) + .expect("decode fixture while collecting HTJ2K jobs"); + assert_eq!( + decoder.jobs.len(), + 14, + "expected every ds0_ht_09_b11 HT block to carry refinement data" + ); + decoder.jobs +} + fn generated_coefficients(width: u32, height: u32, seed: u32) -> Vec { let mut state = seed ^ 0xa24b_aed4; let mut coefficients = Vec::with_capacity(width as usize * height as usize); @@ -30,6 +129,117 @@ fn default_style() -> J2kCodeBlockStyle { } } +fn generated_htj2k_encode_batch(blocks: usize, width: u32, height: u32) -> Vec> { + (0..blocks) + .map(|index| generated_coefficients(width, height, 0xA5A5_0000 ^ index as u32)) + .collect() +} + +fn encoded_ht_block_len( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + let encoded = encode_ht_code_block_scalar(coefficients, width, height, total_bitplanes) + .expect("encode HTJ2K code block"); + encoded.data.len() + + usize::from(encoded.num_coding_passes) + + usize::from(encoded.num_zero_bitplanes) +} + +fn encode_ht_batch_serial( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .iter() + .map(|coefficients| encoded_ht_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn encode_ht_batch_rayon( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .par_iter() + .map(|coefficients| encoded_ht_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn encode_ht_batch_rayon_chunks( + blocks: &[Vec], + chunk_size: usize, + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .par_chunks(chunk_size) + .map(|chunk| encode_ht_batch_serial(chunk, width, height, total_bitplanes)) + .sum() +} + +fn encoded_j2k_block_len( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + let encoded = encode_j2k_code_block_scalar_with_style( + coefficients, + width, + height, + J2kSubBandType::LowLow, + total_bitplanes, + default_style(), + ) + .expect("encode J2K code block"); + encoded.data.len() + + encoded.segments.len() + + usize::from(encoded.number_of_coding_passes) + + usize::from(encoded.missing_bit_planes) +} + +fn encode_j2k_batch_serial( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .iter() + .map(|coefficients| encoded_j2k_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn encode_j2k_batch_rayon( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .par_iter() + .map(|coefficients| encoded_j2k_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn benchmark_thread_counts() -> Vec { + let available = std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1); + let mut counts = vec![1, available.min(2), available.min(4), available]; + counts.sort_unstable(); + counts.dedup(); + counts +} + fn bench_tier1_decode(c: &mut Criterion) { let mut group = c.benchmark_group("tier1_bitplane_decode"); let cases = [ @@ -199,6 +409,67 @@ fn bench_htj2k_cleanup_decode(c: &mut Criterion) { group.finish(); } +fn bench_htj2k_refinement_fixture_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_refinement_fixture_decode"); + let image = + Image::new(HTJ2K_REFINEMENT_FIXTURE, &DecodeSettings::default()).expect("fixture image"); + + group.bench_function("ds0_ht_09_b11_full", |b| { + b.iter(|| { + let mut context = DecoderContext::default(); + let components = image + .decode_components_with_context(&mut context) + .expect("decode HTJ2K refinement fixture"); + black_box(components.planes()[0].samples()); + }); + }); + + group.finish(); +} + +fn bench_htj2k_refinement_block_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_refinement_block_decode"); + let jobs = collect_refinement_fixture_jobs(); + let job = jobs + .first() + .expect("fixture must contain a refinement HTJ2K block"); + let cleanup_job = job.borrowed_with_passes(1, 0); + let sigprop_job = job.borrowed_with_passes(2, job.refinement_length); + let magref_job = job.borrowed_with_passes(job.number_of_coding_passes, job.refinement_length); + + let mut cleanup_output = vec![0.0; job.output_len()]; + group.bench_function("ds0_ht_09_b11_cleanup", |b| { + b.iter(|| { + cleanup_output.fill(0.0); + decode_ht_code_block_scalar(black_box(cleanup_job), &mut cleanup_output) + .expect("cleanup-limited HTJ2K decode"); + black_box(&cleanup_output); + }); + }); + + let mut sigprop_output = vec![0.0; job.output_len()]; + group.bench_function("ds0_ht_09_b11_sigprop", |b| { + b.iter(|| { + sigprop_output.fill(0.0); + decode_ht_code_block_scalar(black_box(sigprop_job), &mut sigprop_output) + .expect("significance-propagation-limited HTJ2K decode"); + black_box(&sigprop_output); + }); + }); + + let mut magref_output = vec![0.0; job.output_len()]; + group.bench_function("ds0_ht_09_b11_magref_full", |b| { + b.iter(|| { + magref_output.fill(0.0); + decode_ht_code_block_scalar(black_box(magref_job), &mut magref_output) + .expect("full HTJ2K decode through magnitude refinement"); + black_box(&magref_output); + }); + }); + + group.finish(); +} + fn bench_htj2k_cleanup_encode(c: &mut Criterion) { let mut group = c.benchmark_group("htj2k_cleanup_encode"); for &seed in &[0x9292_0000, 0x9292_0001] { @@ -227,11 +498,188 @@ fn bench_htj2k_cleanup_encode(c: &mut Criterion) { group.finish(); } +fn bench_htj2k_cleanup_encode_distribution(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode_distribution"); + for &seed in &[0x9292_0000, 0x9292_0001] { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, seed); + + group.bench_with_input( + BenchmarkId::new("rho_eq_uq_64x64", seed), + &coefficients, + |b, coefficients| { + b.iter(|| { + let distribution = collect_ht_cleanup_encode_distribution( + black_box(coefficients), + width, + height, + total_bitplanes, + ) + .expect("collect HTJ2K cleanup encode distribution"); + black_box(distribution); + }); + }, + ); + } + group.finish(); +} + +fn bench_htj2k_cleanup_encode_parallel_granularity(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode_parallel_granularity"); + let width = 64; + let height = 64; + let total_bitplanes = 10; + let blocks = generated_htj2k_encode_batch(128, width, height); + + group.bench_function("serial_128_blocks", |b| { + b.iter(|| { + let total_len = + encode_ht_batch_serial(black_box(&blocks), width, height, total_bitplanes); + black_box(total_len); + }); + }); + + group.bench_function("rayon_par_iter_global_128_blocks", |b| { + b.iter(|| { + let total_len = + encode_ht_batch_rayon(black_box(&blocks), width, height, total_bitplanes); + black_box(total_len); + }); + }); + + for threads in benchmark_thread_counts() { + let pool = ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build benchmark Rayon pool"); + group.bench_with_input( + BenchmarkId::new("rayon_par_iter_threads", threads), + &threads, + |b, _| { + b.iter(|| { + let total_len = pool.install(|| { + encode_ht_batch_rayon(black_box(&blocks), width, height, total_bitplanes) + }); + black_box(total_len); + }); + }, + ); + } + + let available = std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1); + let chunk_pool = ThreadPoolBuilder::new() + .num_threads(available) + .build() + .expect("build benchmark Rayon chunk pool"); + for chunk_size in [4usize, 8, 16, 32, 64] { + group.bench_with_input( + BenchmarkId::new("rayon_par_chunks_128_blocks", chunk_size), + &chunk_size, + |b, &chunk_size| { + b.iter(|| { + let total_len = chunk_pool.install(|| { + encode_ht_batch_rayon_chunks( + black_box(&blocks), + chunk_size, + width, + height, + total_bitplanes, + ) + }); + black_box(total_len); + }); + }, + ); + } + + group.finish(); +} + +fn bench_htj2k_cleanup_encode_parallel_batch_size(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode_parallel_batch_size"); + let width = 64; + let height = 64; + let total_bitplanes = 10; + + for block_count in [1usize, 2, 4, 8, 16, 32, 128] { + let blocks = generated_htj2k_encode_batch(block_count, width, height); + group.bench_with_input( + BenchmarkId::new("serial_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = + encode_ht_batch_serial(black_box(blocks), width, height, total_bitplanes); + black_box(total_len); + }); + }, + ); + group.bench_with_input( + BenchmarkId::new("rayon_par_iter_global_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = + encode_ht_batch_rayon(black_box(blocks), width, height, total_bitplanes); + black_box(total_len); + }); + }, + ); + } + + group.finish(); +} + +fn bench_j2k_tier1_encode_parallel_batch_size(c: &mut Criterion) { + let mut group = c.benchmark_group("j2k_tier1_encode_parallel_batch_size"); + let width = 64; + let height = 64; + let total_bitplanes = 10; + + for block_count in [1usize, 2, 4, 8, 16, 32, 128] { + let blocks = generated_htj2k_encode_batch(block_count, width, height); + group.bench_with_input( + BenchmarkId::new("serial_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = + encode_j2k_batch_serial(black_box(blocks), width, height, total_bitplanes); + black_box(total_len); + }); + }, + ); + group.bench_with_input( + BenchmarkId::new("rayon_par_iter_global_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = + encode_j2k_batch_rayon(black_box(blocks), width, height, total_bitplanes); + black_box(total_len); + }); + }, + ); + } + + group.finish(); +} + criterion_group!( benches, bench_tier1_decode, bench_tier1_encode, bench_htj2k_cleanup_decode, - bench_htj2k_cleanup_encode + bench_htj2k_refinement_fixture_decode, + bench_htj2k_refinement_block_decode, + bench_htj2k_cleanup_encode, + bench_htj2k_cleanup_encode_distribution, + bench_htj2k_cleanup_encode_parallel_granularity, + bench_htj2k_cleanup_encode_parallel_batch_size, + bench_j2k_tier1_encode_parallel_batch_size ); criterion_main!(benches); diff --git a/crates/signinum-j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K b/crates/signinum-j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K new file mode 100644 index 00000000..d896ab36 --- /dev/null +++ b/crates/signinum-j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019 - 2021, Osamu Watanabe +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crates/signinum-j2k-native/fixtures/htj2k/README.md b/crates/signinum-j2k-native/fixtures/htj2k/README.md new file mode 100644 index 00000000..1c5d7e6f --- /dev/null +++ b/crates/signinum-j2k-native/fixtures/htj2k/README.md @@ -0,0 +1,22 @@ +HTJ2K fixtures for native decoder coverage. + +These fixtures are from the OpenHTJ2K conformance data, copied from OpenHTJ2K +commit `ffe5acf9f1eedb87c36c3fd2134fdc1ddea5e75f`. They are tiny HTONLY +codestreams derived from the JPEG 2000 Part 4 / ITU-T T.803 HTJ2K conformance +set. + +`openhtj2k_ds0_ht_12_b11.j2k` is copied from `ds0_ht_12_b11.j2k`, blob +`cf3fb0bc7e55898b4e6977f38ba0d38d91c359bf`. The native decoder sees 8 HT code +blocks, 2 non-empty refinement jobs, and up to 3 HT coding passes. + +`openhtj2k_ds0_ht_09_b11.j2k` is copied from `ds0_ht_09_b11.j2k`, blob +`d4f2031359c32eb24825d00dde05a92cf3ae451e`. The native decoder sees 14 HT +code blocks, all with non-empty refinement jobs, and up to 3 HT coding passes. + +It is intentionally checked-in test data: tests must not invoke an external +encoder or decoder at runtime. + +The paired `.gray` file contains the expected 8-bit grayscale samples in +row-major order from decoding the checked-in codestream with OpenJPH. + +The OpenHTJ2K source license is retained in `LICENSE.OpenHTJ2K`. diff --git a/crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray b/crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray new file mode 100644 index 0000000000000000000000000000000000000000..115938f1ecc4e1476911c9bf0730ff5f5f582beb GIT binary patch literal 629 ycmZo@7+!$k|9=Js5CQWRkozAb_8+GJh8DsC7={wof}sM#3=9DbEif+(bs7Nm(m2-u literal 0 HcmV?d00001 diff --git a/crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k b/crates/signinum-j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k new file mode 100644 index 0000000000000000000000000000000000000000..d4f2031359c32eb24825d00dde05a92cf3ae451e GIT binary patch literal 790 zcmezG|38pH(}4j91c6u;#DD@+J|jCLOIL**r!oYBV@f0%yp9V z$L>-F=5yx~HZ$E!804G*=X`xNqaT;097~fUo4>19q!|56p850vH%tAHRxW6;OC_VWBfS zi$|k2-#QBo28Is|OAgP7bMaJ3vDy~PVCKWfn)fz@`9Y!%%b7+)U}z|vksJ+AX8K_Qob9UImqdEL91e{hVyyq<(V&U37+8Ka6O^URMuak#G!1&67%EF4F={0 zsS+iV7cSZL6Wz)t{%B*hO6v5z1x#5C=_x%se(Y&^nxFcFvDt9j7Nbp)GnY=4WNuTv zJZbBV8!R_jw%*v1GR?=eb@}qvqsNaQZ*Of~p6=?})|#{0*L#BRgkIn7oa{DM=K~VX zt6Ov0a@